Use Dataset created from 02_CLS_Data_Summary_2022_0914_Data_Analysis File

Loading Data

Load Google Sheet

Final_CLS_2022_Study_List_Non_Search_model_file <- read_sheet(
  "https://docs.google.com/spreadsheets/d/1N48rTeq7md0v8w8pG_8XIiuapPHQAeO5WoWIB3eaceI/edit#gid=1449351377",
  sheet = "FinalDataset_2022_Update"
) %>%
  mutate(
    Significant_Spend =
      as.numeric(
        case_when(
          probability_of_lift >= 0.9 ~ 1,
          TRUE ~ 0
        )
      ),
    country = case_when(
      country == "NA" ~ "US",
      TRUE ~ country
    ),
    region_v2 = case_when(
      country == "US" ~ "NA",
      country == "CA" ~ "NA",
      country == "US + CA" ~ "NA",
      TRUE ~ region
    )
  ) %>%
  filter(channel != "Search") %>%
  # filter out studies without reported lifts
  filter(exposed != -1) %>%
  # filter out google pay study
  filter(study_id != "149142217") %>%
  # filter out very negative absolute lifts
  filter(absolute_lift > -1000) %>%
  mutate(
    pa = case_when(
      pa == "Google Ads" ~ "SMB", # Step 1
      pa == "YouTube" & conversion != "Type 256522942 ([MCC] YouTube TV - Web - Trial Start)" ~ "YTMP", # Step 2
      pa == "YouTube Premium" ~ "YTMP", # Step 2
      conversion == "Type 256522942 ([MCC] YouTube TV - Web - Trial Start)" ~ "YouTube TV", # Step 2
      pa == "Cloud" & conversion != "Type 14257803 (Enterprise - Apps - Signup Confirm - Unique)" ~ "Cloud Workspace", # Step 3
      pa == "Cloud" & conversion == "Type 14257803 (Enterprise - Apps - Signup Confirm - Unique)" ~ "Cloud GCP", # Step 3
      pa == "Project Fi" ~ "Google Fi", # Step 4
      pa == "Google Chrome" ~ "Chrome",
      TRUE ~ pa
    )
  ) %>%
  mutate(
    parsed_type = parse_number(conversion),
    grouped_conversion = case_when(
      conversion %in% c("Chromebook Microsite Referral Clicks Q4 2015", "Type 251422729 (Chromebooks Microsite Referral Clicks (Q4 2017))") ~ "Chromebook Referrals",
      conversion %in% c("Desktop Downloads", "Type 11541547 (Desktop Download)") ~
        "Desktop Downloads",
      pa == "Pixel" ~ "Mobile Conversions",
      pa == "DSM" ~ "Non-Mobile Device Conversions",
      conversion == "Type 302982954 (Lena - P Lead)" ~ "Lena P Lead",
      conversion == "Type 288347008 (LENA - B Lead)" ~ "Lena B Lead",
      conversion == "Type 288697653 (LENA - Q Lead)" ~ "Lena Q Lead",
      parsed_type %in% c(181283993, 855508686) ~ "Workspace Free Trial Start",
      parsed_type == 330755641 ~ "Microsite Conversions",
      parsed_type == 14257803 ~ "Enterprise Signups",
      parsed_type == 289680712 ~ "Google(iOs) First Open",
      parsed_type == 256522942 ~ "YouTube TV - Web - Trial Start",
      parsed_type %in% c(452391534, 221497833, 277150074) ~ "Trial Signups Complete",
      TRUE ~ conversion
    ),
    pa = case_when(
      conversion == "Type 288697653 (LENA - Q Lead)" ~ "SMB-QLead",
      TRUE ~ pa
    )
  ) %>%
  filter(absolute_lift > 0)


# all.equal(Final_CLS_2022_Study_List_Non_Search_model_file,Final_CLS_2022_Study_List_Non_Search_v3)

Create All Response Curves only normal powers

Folder for all Output and scripts

file.sources <- list.files(path = "RScripts/", pattern = "*.R", full.names = TRUE)
sapply(file.sources, source, .GlobalEnv)
        RScripts/best_ind_function.R RScripts/export_rplots_function.R RScripts/export_rplots_function2.R
value   ?                            ?                                 ?                                 
visible FALSE                        FALSE                             FALSE                             
        RScripts/graphing_function.R RScripts/graphing_function_elasticnet.R RScripts/graphing_function_rlm.R
value   ?                            ?                                       ?                               
visible FALSE                        FALSE                                   FALSE                           
        RScripts/graphing_function2.R RScripts/graphing_function3.R RScripts/graphing_function4.R
value   ?                             ?                             ?                            
visible FALSE                         FALSE                         FALSE                        
        RScripts/graphing_function4_w_anom.R RScripts/model_wrapper_function.R RScripts/model_wrapper_function2.R
value   ?                                    ?                                 ?                                 
visible FALSE                                FALSE                             FALSE                             
        RScripts/named_group_split.R RScripts/names_function.R RScripts/ridge_lasso_function.R RScripts/ridge_lasso_function2.R
value   ?                            ?                         ?                               ?                               
visible FALSE                        FALSE                     FALSE                           FALSE                           
        RScripts/ridge_lasso_function4.R RScripts/rlm_function.R RScripts/rlm_function2.R
value   ?                                ?                       ?                       
visible FALSE                            FALSE                   FALSE                   

Check parameters


### powers to try
powers <- seq(0.1, 0.9, by = 0.01)
powers2 <- 1

### Powers to Try
#powers <- seq(0.1, 0.9, by = 0.01)
#powers2 <-seq(1.5,3, by = 0.25)


### Lambda parameters
parameters <- c(
  #  seq(0.1, 2, by =0.1) ,  seq(2, 5, 0.5) ,
  seq(5, 29, 1)
  ,seq(30, 102, 4)
  ,seq(110, 1000, 15)
  ,seq(1000, 10020, 500)
)

### elasticnet parameters
alpha_parameters <- c(seq(0, 1, 0.25))

# For Testing Purposes
#alpha_parameters <- c(seq(1, 1, 1))

Testing Different Model Types

Chrome

Data Readin


start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_chrome_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa == "Chrome") %>%
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_chrome_pre %>%
  select(
    region_v2, country, channel, tactic,
    cost_spent_on_exposed_group:absolute_lift
  )

iso_chrome <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1153)

iso_chrome$fit(df_test)

scores_train <- df_test %>%
  iso_chrome$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_chrome_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_chrome_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>% 
  filter(average_depth > 3)


Final_CLS_2022_Study_List_Non_Search_model_file_chrome <-
  Final_CLS_2022_Study_List_Non_Search_model_file_chrome_pre2 %>%
  named_group_split(tactic)

Run Model



fits_non_search_chrome <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_chrome,poly_ind = 0)

best_ind_non_search_chrome <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_chrome), best_ind_function,df = fits_non_search_chrome,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_chrome) 

coef_non_search_chrome <- best_ind_non_search_chrome %>% bind_rows #make a matrix of all coefs

best_fit_non_search_chrome <- best_ind_non_search_chrome %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_chrome_pre, tactic))  

Create Graph Object

graph_list_chrome <- lapply(1:length(best_fit_non_search_chrome), graphing_function4, df1 = best_fit_non_search_chrome, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_chrome)
end_time <- Sys.time()

time_chrome = end_time - start_time

time_chrome

Cloud

Data Readin

start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_cloud_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa %in% c("Cloud GCP", "Cloud Workspace")) %>%
  mutate(
    pa = "Cloud",
    pa2 = "Cloud - All Channel"
  ) %>%
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_cloud_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift, parsed_type
  )

iso_cloud <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1153)

iso_cloud$fit(df_test)

scores_train <- df_test %>%
  iso_cloud$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_cloud_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_cloud_pre %>%
  left_join(scores_train, by = c("id2" = "id"))

Final_CLS_2022_Study_List_Non_Search_model_file_cloud <-
  Final_CLS_2022_Study_List_Non_Search_model_file_cloud_pre2 %>%
  named_group_split(pa2)

Run Model

fits_non_search_cloud <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_cloud,poly_ind = 0)

best_ind_non_search_cloud <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_cloud), best_ind_function,df = fits_non_search_cloud,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_cloud) 

coef_non_search_cloud <- best_ind_non_search_cloud %>% bind_rows #make a matrix of all coefs

best_fit_non_search_cloud <- best_ind_non_search_cloud %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_cloud_pre, pa2))  

Create Graph Object

graph_list_cloud <- lapply(1:length(best_fit_non_search_cloud), graphing_function4, df1 = best_fit_non_search_cloud, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_cloud)
end_time <- Sys.time()

time_cloud = end_time - start_time

YouTube

Data Readin


start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_youtube_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa %in% c("YouTube TV", "YTMP")) %>%
  mutate(
    pa = "YouTube",
    pa2 = "YouTube"
  ) %>%
  #  filter(absolute_lift < 5000) %>%
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_youtube_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift, parsed_type
  )

iso_yt <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1153)

iso_yt$fit(df_test)

scores_train <- df_test %>%
  iso_yt$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_youtube_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_youtube_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>%
  filter(average_depth > 3.89)

Final_CLS_2022_Study_List_Non_Search_model_file_youtube <-
  Final_CLS_2022_Study_List_Non_Search_model_file_youtube_pre2 %>%
  named_group_split(region_v2)

Run Model

fits_non_search_youtube <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_youtube,poly_ind = 0)

best_ind_non_search_youtube <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_youtube), best_ind_function,df = fits_non_search_youtube,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_youtube) 

coef_non_search_youtube <- best_ind_non_search_youtube %>% bind_rows #make a matrix of all coefs

best_fit_non_search_youtube <- best_ind_non_search_youtube %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_youtube_pre, pa2))  

Create Graph Object

graph_list_youtube <- lapply(1:length(best_fit_non_search_youtube), graphing_function4, df1 = best_fit_non_search_youtube, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_youtube)
end_time <- Sys.time()

time_youtube = end_time - start_time

DSM

Data Readin


start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa == "DSM") %>%
  filter(region_v2 != "APAC") %>%
  # filter(absolute_lift < 1000) # %>%
  # filter(study_id != '6297420') #%>%
  #  filter(study_id !='149161711') %>%
  #  filter(study_id != '148613002') %>%
  # filter(study_id !='3284625') %>%
  #  filter(study_id !='3329131')
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift
  )

iso_dsm <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1152)

iso_dsm$fit(df_test)

scores_train <- df_test %>%
  iso_dsm$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>%
  filter(average_depth > 5)

Final_CLS_2022_Study_List_Non_Search_model_file_dsm <-
  Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre2 %>%
  named_group_split(region_v2, channel)

Run Model

fits_non_search_dsm <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_dsm,poly_ind = 0)

best_ind_non_search_dsm <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_dsm), best_ind_function,df = fits_non_search_dsm,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_dsm)

coef_non_search_dsm <- best_ind_non_search_dsm %>% bind_rows #make a matrix of all coefs

best_fit_non_search_dsm <- best_ind_non_search_dsm %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre, region_v2,channel))  

Create Graph Object

graph_list_dsm <- lapply(1:length(best_fit_non_search_dsm), graphing_function4, df1 = best_fit_non_search_dsm, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_dsm)
end_time <- Sys.time()

time_dsm = end_time - start_time

Pixel

Data Readin


start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_pixel_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa == "Pixel") %>%
  mutate(
    pa2 = "Pixel - All Channel"
  ) %>%
  #   filter(absolute_lift < 1000)  %>%
  # filter(study_id != '6297420') #%>%
  #  filter(study_id !='149161711') %>%
  #  filter(study_id != '148613002') %>%
  # filter(study_id !='3284625') %>%
  #  filter(study_id !='3329131')
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_pixel_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift
  )

iso_pixel <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1152)

iso_pixel$fit(df_test)

scores_train <- df_test %>%
  iso_pixel$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_pixel_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_pixel_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>%
  filter(average_depth > 3.1)

Final_CLS_2022_Study_List_Non_Search_model_file_pixel_pre2


Final_CLS_2022_Study_List_Non_Search_model_file_pixel <-
  Final_CLS_2022_Study_List_Non_Search_model_file_pixel_pre2 %>%
  named_group_split(pa2)

Run Model

fits_non_search_pixel <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_pixel,poly_ind = 0)

best_ind_non_search_pixel <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_pixel), best_ind_function,df = fits_non_search_pixel,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_pixel) 

coef_non_search_pixel <- best_ind_non_search_pixel %>% bind_rows #make a matrix of all coefs

best_fit_non_search_pixel <- best_ind_non_search_pixel %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_pixel_pre, pa2))  

Create Graph Object

graph_list_pixel <- lapply(1:length(best_fit_non_search_pixel), graphing_function4, df1 = best_fit_non_search_pixel, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_pixel)
end_time <- Sys.time()

time_pixel = end_time - start_time

Fi

Data Readin


start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_fi_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa == "Google Fi") %>%
  mutate(
    pa2 = "Fi - All Channel"
  ) %>%
  #   filter(absolute_lift < 1000)  %>%
  # filter(study_id != '6297420') #%>%
  #  filter(study_id !='149161711') %>%
  #  filter(study_id != '148613002') %>%
  # filter(study_id !='3284625') %>%
  #  filter(study_id !='3329131')
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_fi_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift
  )

iso_fi <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1152)

iso_fi$fit(df_test)

scores_train <- df_test %>%
  iso_fi$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_fi_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_fi_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>%
  filter(average_depth > 4.75)

Final_CLS_2022_Study_List_Non_Search_model_file_fi_pre2


Final_CLS_2022_Study_List_Non_Search_model_file_fi <-
  Final_CLS_2022_Study_List_Non_Search_model_file_fi_pre2 %>%
  named_group_split(channel)

Run Model

fits_non_search_fi <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_fi,poly_ind = 0)

best_ind_non_search_fi <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_fi), best_ind_function,df = fits_non_search_fi,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_fi) 

coef_non_search_fi <- best_ind_non_search_fi %>% bind_rows #make a matrix of all coefs

best_fit_non_search_fi <- best_ind_non_search_fi %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_fi_pre, pa2))  

Create Graph Object

graph_list_fi <- lapply(1:length(best_fit_non_search_fi), graphing_function4, df1 = best_fit_non_search_fi, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_fi)
end_time <- Sys.time()

time_fi = end_time - start_time

SMB - QLeads

Data Readin


start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_smbq_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(grouped_conversion == 'Lena Q Lead') %>%
  mutate(
    pa2 = "SMB - Q-Lead"
  ) %>%
  #   filter(absolute_lift < 1000)  %>%
  # filter(study_id != '6297420') #%>%
  #  filter(study_id !='149161711') %>%
  #  filter(study_id != '148613002') %>%
  # filter(study_id !='3284625') %>%
  #  filter(study_id !='3329131')
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_smbq_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift
  )

iso_smbq <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1152)

iso_smbq$fit(df_test)

scores_train <- df_test %>%
  iso_smbq$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_smbq_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_smbq_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>% 
  filter(average_depth > 1)

Final_CLS_2022_Study_List_Non_Search_model_file_smbq <-
  Final_CLS_2022_Study_List_Non_Search_model_file_smbq_pre2 %>%
  named_group_split(pa2)

Run Model

fits_non_search_smbq <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_smbq,poly_ind = 0)

best_ind_non_search_smbq <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_smbq), best_ind_function,df = fits_non_search_smbq,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_smbq) 

coef_non_search_smbq <- best_ind_non_search_smbq %>% bind_rows #make a matrix of all coefs

best_fit_non_search_smbq <- best_ind_non_search_smbq %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_smbq_pre, pa2))  

Create Graph Object

graph_list_smbq <- lapply(1:length(best_fit_non_search_smbq), graphing_function4, df1 = best_fit_non_search_smbq, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_smbq)
end_time <- Sys.time()

time_smbq = end_time - start_time

SMB - BLeads

Data Readin


start_time <- Sys.time()

Final_CLS_2022_Study_List_Non_Search_model_file_smbb_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa == "SMB" & grouped_conversion == 'Lena B Lead') %>%
  mutate(
    pa2 = "SMB - B-Lead"
  ) %>%
  #   filter(absolute_lift < 1000)  %>%
  # filter(study_id != '6297420') #%>%
  #  filter(study_id !='149161711') %>%
  #  filter(study_id != '148613002') %>%
  # filter(study_id !='3284625') %>%
  #  filter(study_id !='3329131')
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_smbb_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift
  )

iso_smbb <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1152)

iso_smbb$fit(df_test)

scores_train <- df_test %>%
  iso_smbb$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_smbb_pre2 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_smbb_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>% 
  filter(average_depth > 4)

Final_CLS_2022_Study_List_Non_Search_model_file_smbb <-
  Final_CLS_2022_Study_List_Non_Search_model_file_smbb_pre2 %>%
  named_group_split(channel)

Run Model

fits_non_search_smbb <- model_wrapper_function(df = Final_CLS_2022_Study_List_Non_Search_model_file_smbb,poly_ind = 0)

best_ind_non_search_smbb <- 
  lapply(1:length(Final_CLS_2022_Study_List_Non_Search_model_file_smbb), best_ind_function,df = fits_non_search_smbb,
         df2 = Final_CLS_2022_Study_List_Non_Search_model_file_smbb) 

coef_non_search_smbb <- best_ind_non_search_smbb %>% bind_rows #make a matrix of all coefs

best_fit_non_search_smbb <- best_ind_non_search_smbb %>%
  set_names(names_function(Final_CLS_2022_Study_List_Non_Search_model_file_smbb_pre, channel))  

Create Graph Object

graph_list_smbb <- lapply(1:length(best_fit_non_search_smbb), graphing_function4, df1 = best_fit_non_search_smbb, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_smbb)
end_time <- Sys.time()

time_smbb = end_time - start_time

Export all graph lists

graph_names <- mget(ls(pat = 'graph_list_'))
   
df_names <- mget(setdiff(ls(pattern = 'Final_CLS_2022_Study_List_Non_Search_model_file_'), ls(pattern = "pre")))

#rm(Final_CLS_2022_Study_List_Non_Search_model_file_Chrome,Final_CLS_2022_Study_List_Non_Search_model_file_Cloud,Final_CLS_2022_Study_List_Non_Search_model_file_YouTube)

#lapply(1:length(graph_names),
#      function(j) {
#lapply(1:length(df_names[[j]]),export_rplots_function2,starting_name = "Non_Search_",folder_name = folder_name,df_list = #df_names[[j]],graphing_list = graph_names[j][[1]])
#      }
#       )

Grid of all Response Curves

Sub Plot Documentation

Coef Matrix

Graphs with Anomaly Scores

graph_list.fi <- lapply(1:length(best_fit_non_search_fi), graphing_function4_w_anom, df1 = best_fit_non_search_fi, df2 = Final_CLS_2022_Study_List_Non_Search_model_file_fi)

### Add GG Text Repel
ggplotly(graph_list.fi[[3]])

Create all Response Curves - Ridge/Lasso


start_time <- Sys.time()

fits.non.search.RIDGE_LASSO <- lapply(
  1:length(df_names),
  function(i) {
    model_wrapper_function(df = df_names[i][[1]],poly_ind = 0)
  }
)

Create all Response Curves - RLM



start_time <- Sys.time()

fits.non.search.RLM <- lapply(
  1:length(df_names),
  function(i) {
    model_wrapper_function2(df = df_names[i][[1]])
  }
)

end_time <- Sys.time()

combined_rlm_time <- start_time - end_time

best.ind.non.search.RLM <- lapply(
  1:length(df_names),
  function(i) {   
  lapply(1:length(df_names[i][[1]]), best_ind_function,df = fits.non.search.RLM[i][[1]],
         df2 = df_names[i][[1]])
  }
)

coef.non.search.RLM <- lapply(
  1:length(df_names),
  function (i){
  best.ind.non.search.RLM[i][[1]] %>% bind_rows
  }
) %>%
  bind_rows() %>% 
  as.data.frame() %>% 
  mutate(
    cost_p2 = 0,
    lambda = 0,
    alpha = 0,
    powers2 = 0
  ) %>% 
  select(one_of(colnames(coef.2_matrix)))


best.fit.non.search.RLM <- lapply(1:length(df_names),
      function(j) {
lapply(1:length(best.ind.non.search.RLM[[j]]),
      function(i){
        best.ind.non.search.RLM[j][[1]][i] %>% 
        set_names(nm = best.ind.non.search.RLM[j][[1]][[i]]["model"])
      } 
)
      }
       )


  

-combined_ridge_time+combined_rlm_time
Time difference of 32.768 mins

graph.list.rlm <- lapply(1:length(df_names),
      function(i){
      lapply(1:length(best.fit.non.search.RLM[i][[1]]), graphing_function_rlm, df1= best.fit.non.search.RLM[i],df2 = df_names[i])
      } 
)


graph.list.RIDGE_LASSO <- lapply(1:length(df_names),
      function(i){
      lapply(1:length(best.fit.non.search.RIDGE_LASSO[i][[1]]), graphing_function_elasticnet, df1= best.fit.non.search.RIDGE_LASSO[i],df2 = df_names[i])
      } 
)

Export all Plots


folder_name1 <- paste0("Output/", "outputfiles_", Sys.Date(), "_", "RLM", "/")
dir.create(folder_name1) # it will throw a warning if folder exists
Warning in dir.create(folder_name1) :
  'Output\outputfiles_2022-11-08_RLM' already exists
lapply(1:length(df_names),
      function(j) {
lapply(1:length(df_names[[j]]),export_rplots_function2,starting_name = "Non_Search_",folder_name = folder_name1,df_list = df_names[[j]],graphing_list = graph.list.rlm[j][[1]])
      }
       )
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
[[1]]
[[1]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Chrome_All_Channel.png"

[[1]][[2]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Chrome_non-REMK.png"

[[1]][[3]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Chrome_REMK.png"


[[2]]
[[2]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Cloud_Cloud_-_All_Channel.png"


[[3]]
[[3]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_DSM_EMEA__DISCOVERY.png"

[[3]][[2]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_DSM_EMEA__DISPLAY.png"

[[3]][[3]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_DSM_EMEA__YOUTUBE.png"

[[3]][[4]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_DSM_NA__DISCOVERY.png"

[[3]][[5]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_DSM_NA__DISPLAY.png"

[[3]][[6]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_DSM_NA__YOUTUBE.png"


[[4]]
[[4]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Google_Fi_DISCOVERY.png"

[[4]][[2]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Google_Fi_DISPLAY.png"

[[4]][[3]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Google_Fi_YOUTUBE.png"


[[5]]
[[5]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_Pixel_Pixel_-_All_Channel.png"


[[6]]
[[6]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_SMB_DISCOVERY.png"

[[6]][[2]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_SMB_DISPLAY.png"

[[6]][[3]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_SMB_YOUTUBE.png"


[[7]]
[[7]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_SMB-QLead_SMB_-_Q-Lead.png"


[[8]]
[[8]][[1]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_YouTube_APAC.png"

[[8]][[2]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_YouTube_EMEA.png"

[[8]][[3]]
[1] "Output/outputfiles_2022-11-08_RLM/Non_Search_YouTube_NA.png"

folder_name2 <- paste0("Output/", "outputfiles_", Sys.Date(), "_", "ElasticNet", "/")
dir.create(folder_name2) # it will throw a warning if folder exists
Warning in dir.create(folder_name2) :
  'Output\outputfiles_2022-11-08_ElasticNet' already exists
lapply(1:length(df_names),
      function(j) {
lapply(1:length(df_names[[j]]),export_rplots_function2,starting_name = "Non_Search_",folder_name = folder_name2,df_list = df_names[[j]],graphing_list = graph.list.RIDGE_LASSO[j][[1]])
      }
       )
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
Saving 18.8 x 12.5 in image
[[1]]
[[1]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Chrome_All_Channel.png"

[[1]][[2]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Chrome_non-REMK.png"

[[1]][[3]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Chrome_REMK.png"


[[2]]
[[2]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Cloud_Cloud_-_All_Channel.png"


[[3]]
[[3]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_DSM_EMEA__DISCOVERY.png"

[[3]][[2]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_DSM_EMEA__DISPLAY.png"

[[3]][[3]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_DSM_EMEA__YOUTUBE.png"

[[3]][[4]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_DSM_NA__DISCOVERY.png"

[[3]][[5]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_DSM_NA__DISPLAY.png"

[[3]][[6]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_DSM_NA__YOUTUBE.png"


[[4]]
[[4]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Google_Fi_DISCOVERY.png"

[[4]][[2]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Google_Fi_DISPLAY.png"

[[4]][[3]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Google_Fi_YOUTUBE.png"


[[5]]
[[5]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_Pixel_Pixel_-_All_Channel.png"


[[6]]
[[6]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_SMB_DISCOVERY.png"

[[6]][[2]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_SMB_DISPLAY.png"

[[6]][[3]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_SMB_YOUTUBE.png"


[[7]]
[[7]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_SMB-QLead_SMB_-_Q-Lead.png"


[[8]]
[[8]][[1]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_YouTube_APAC.png"

[[8]][[2]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_YouTube_EMEA.png"

[[8]][[3]]
[1] "Output/outputfiles_2022-11-08_ElasticNet/Non_Search_YouTube_NA.png"

Show Graphs

lapply(1:length(df_names),
function(j){
  subplot(graph.list.rlm[j][[1]], nrows = length(graph.list.rlm[j][[1]]))
}
)
[[1]]

[[2]]

[[3]]

[[4]]

[[5]]

[[6]]

[[7]]

[[8]]
lapply(1:length(df_names),
function(i){
#p1 = graph_names[i][[1]]
do.call(grid.arrange,graph.list.rlm[i][[1]])
#return(grid.arrange(grobs = p1))
}
)
[[1]]
TableGrob (3 x 1) "arrange": 3 grobs

[[2]]
TableGrob (1 x 1) "arrange": 1 grobs

[[3]]
TableGrob (3 x 2) "arrange": 6 grobs

[[4]]
TableGrob (3 x 1) "arrange": 3 grobs

[[5]]
TableGrob (1 x 1) "arrange": 1 grobs

[[6]]
TableGrob (3 x 1) "arrange": 3 grobs

[[7]]
TableGrob (1 x 1) "arrange": 1 grobs

[[8]]
TableGrob (3 x 1) "arrange": 3 grobs
NA

lapply(1:length(df_names),
function(j){
  subplot(graph.list.RIDGE_LASSO[j][[1]], nrows = length(graph.list.RIDGE_LASSO[j][[1]]))
}
)
[[1]]

[[2]]

[[3]]

[[4]]

[[5]]

[[6]]

[[7]]

[[8]]
lapply(1:length(df_names),
function(i){
#p1 = graph_names[i][[1]]
do.call(grid.arrange,graph.list.RIDGE_LASSO[i][[1]])
#return(grid.arrange(grobs = p1))
}
)
[[1]]
TableGrob (3 x 1) "arrange": 3 grobs

[[2]]
TableGrob (1 x 1) "arrange": 1 grobs

[[3]]
TableGrob (3 x 2) "arrange": 6 grobs

[[4]]
TableGrob (3 x 1) "arrange": 3 grobs

[[5]]
TableGrob (1 x 1) "arrange": 1 grobs

[[6]]
TableGrob (3 x 1) "arrange": 3 grobs

[[7]]
TableGrob (1 x 1) "arrange": 1 grobs

[[8]]
TableGrob (3 x 1) "arrange": 3 grobs
NA

Testing Metafor Package


p_load(lme4)
p_load(metaforest)

Testing on DSM Data

Load in Data



Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre <-
  Final_CLS_2022_Study_List_Non_Search_model_file %>%
  filter(pa == "DSM") %>%
  filter(region_v2 != "APAC") %>%
  # filter(absolute_lift < 1000) # %>%
  # filter(study_id != '6297420') #%>%
  #  filter(study_id !='149161711') %>%
  #  filter(study_id != '148613002') %>%
  # filter(study_id !='3284625') %>%
  #  filter(study_id !='3329131')
  mutate(
    id2 = row_number()
  )

df_test <-
  Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre %>%
  # select(-study_id, -id2, -region, -scaling_factor, -quarter, -pa, -study_name)
  select(
    region_v2, country, channel, tactic,
    # treatment_user_count:control,
    cost_spent_on_exposed_group:absolute_lift
  )

iso_dsm <- isolationForest$new(sample_size = nrow(df_test), num_trees = 10000, seed = 1152)

iso_dsm$fit(df_test)
INFO  [19:29:47.734] Building Isolation Forest ...
INFO  [19:29:47.957] done
INFO  [19:29:47.967] Computing depth of terminal nodes ...
INFO  [19:30:04.726] done
INFO  [19:30:04.906] Completed growing isolation forest
scores_train <- df_test %>%
  iso_dsm$predict() %>%
  arrange(desc(anomaly_score))

Final_CLS_2022_Study_List_Non_Search_model_file_dsm_Meta4_V1 <-
  Final_CLS_2022_Study_List_Non_Search_model_file_dsm_pre %>%
  left_join(scores_train, by = c("id2" = "id")) %>%
 filter(average_depth > 5.209)


Final_CLS_2022_Study_List_Non_Search_model_file_dsm_Meta4 <-
Final_CLS_2022_Study_List_Non_Search_model_file_dsm_Meta4_V1 %>% 
  mutate(
    p1 = exposed/treatment_user_count,
    q1 = 1 - p1,
    n1 = treatment_user_count,
    sd1 = sqrt(p1*q1*n1),
    p2 = scaled_control/treatment_user_count,
    q2 = 1-p2,
    n2 = treatment_user_count, 
    sd2 = sqrt(p2*q2*n2),
    cost_p = cost_spent_on_exposed_group ^ 0.4
  ) %>% 
  select(-p1,-q1,-n1,-p2,-q2,-n2) %>% 
  named_group_split(pa)

Standardize Data and after calculating standard deviation


df_SMD <- list()
for (i in 1:length(Final_CLS_2022_Study_List_Non_Search_model_file_dsm_Meta4)){
  df_SMD[[i]] <-
  escalc(
    measure = "SMD",
                 m1i = exposed,
                 m2i = scaled_control,
                 sd1i = sd1,
                 sd2i = sd2,
                 n1i = treatment_user_count,
                 n2i = treatment_user_count,
    data = Final_CLS_2022_Study_List_Non_Search_model_file_dsm_Meta4[[i]]
  )
  names(df_SMD)[i] <- names(Final_CLS_2022_Study_List_Non_Search_model_file_dsm_Meta4[i])
}

Run a Fixed-Effects Model

Documentation




i = 1

yi_DSM = df_SMD[[i]]['absolute_lift'] %>% unlist()
vi_DSM = df_SMD[[i]]['sd1'] %>% unlist()
split2 = factor(df_SMD[[i]]['channel'] %>% unlist(),labels = unique(df_SMD[[i]]['channel']) %>% unlist())


m_reg <- rma(yi = yi,     # The d-column of the df, which contains Cohen's d
         vi = vi   # The vi-column of the df, which contains the variances
       ,mods = ~channel:cost_p-1 #to remove intercept between slopes
       ,data = df_SMD[[i]]
         )  
       
m_reg

Mixed-Effects Model (k = 23; tau^2 estimator: REML)

tau^2 (estimated amount of residual heterogeneity):     21.6012 (SE = 6.8309)
tau (square root of estimated tau^2 value):             4.6477
I^2 (residual heterogeneity / unaccounted variability): 100.00%
H^2 (unaccounted variability / sampling variability):   49986503.17

Test for Residual Heterogeneity:
QE(df = 20) = 136662595.7290, p-val < .0001

Test of Moderators (coefficients 1:3):
QM(df = 3) = 25.5645, p-val < .0001

Model Results:

---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
#rm(Final_CLS_2022_Study_List_Non_Search_model_file_dsm_Meta4_V1)#,mod1_test, i)

predict(m_reg)
forest(m_reg, slab = df_SMD[[i]]['study_name'] %>% unlist(), addcred = TRUE)

Additional Test


# Specify basic plot, mapping sex to the x-axis, effect size 'd' to the y-axis,
# and 'weights' to the weight parameter.

df_SMD[[i]] %>% 
  ggplot()+
  aes(
    x = cost_spent_on_exposed_group,
    y = yi,
    size = 1/sqrt(vi)
  ) +
  geom_point(shape = 1) + # Add scatter
  geom_abline(intercept = 0, slope = m_reg$b[2]) + # Add regression line
 # theme_bw() + # Apply black and white theme
  theme(legend.position = "none") # Remove legend

Mixed Effects Model

Documentation: * https://pages.stat.wisc.edu/~bates/UseR2008/WorkshopD.pdf


i = 1

mod_lme4 <- lmer(formula = log(absolute_lift) ~ 0 + cost_p
       #          + region_v2
             #     + channel 
    #               + (0+ region_v2|channel)
                   + (1 + cost_p|channel) 
     #              + (0+ 1|channel)
         #          + (1+ 1|channel:tactic)
      #             + (channel|tactic)
    #               +(cost_p:channel)
               ,data = df_SMD[[i]], REML =  TRUE) #False calls on MLE which are known to be biased
boundary (singular) fit: see ?isSingular
summary(mod_lme4)
Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log(absolute_lift) ~ 0 + cost_p + (1 + cost_p | channel)
   Data: df_SMD[[i]]

REML criterion at convergence: 88.6

Scaled residuals: 
   Min     1Q Median     3Q    Max 
-2.198 -0.150  0.142  0.550  2.074 

Random effects:
 Groups   Name        Variance Std.Dev. Corr 
 channel  (Intercept) 15.25205 3.9054        
          cost_p       0.00111 0.0333   -1.00
 Residual              1.48538 1.2188        
Number of obs: 23, groups:  channel, 3

Fixed effects:
       Estimate Std. Error      df t value  Pr(>|t|)    
cost_p  0.04817    0.00316 5.79695    15.3 0.0000067 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
optimizer (nloptwrap) convergence code: 0 (OK)
boundary (singular) fit: see ?isSingular
predict(mod_lme4)
     1      2      3      4      5      6      7      8      9     10     11     12     13     14     15 
5.5910 5.4268 5.3498 4.8830 5.6498 5.8099 4.3085 4.2552 3.8610 4.3744 4.9673 5.9422 6.1465 3.9900 5.1042 
    16     17     18     19     20     21     22     23 
3.6101 5.1936 5.1127 5.8055 4.6467 4.6814 5.6567 4.9926 
df_SMD[[i]]$preds <- predict(mod_lme4)

fixef(mod_lme4)
  cost_p 
0.048166 
ranef(mod_lme4, drop = FALSE)
$channel
          (Intercept)     cost_p
DISCOVERY     5.90082 -0.0502700
DISPLAY       0.98253 -0.0083704
YOUTUBE       3.49158 -0.0297453

with conditional variances for “channel” 
p<-
df_SMD[[i]] %>%   
  ggplot(aes(x=cost_spent_on_exposed_group, y=preds, group = channel, colour = channel)) +
  geom_line() + 
  labs(x="Spend", y="Absolute Lift") +
  ggtitle("Mixed Effects Model") + 
#  scale_colour_discrete('pa')+
  geom_jitter(aes(x=cost_spent_on_exposed_group, y = log(absolute_lift), size = vi )) 

p


ggplotly(p)
NA

Updated Plotting Function

https://lmudge13.github.io/sample_code/mixed_effects.html

p_load(sjPlot) #for plotting lmer and glmer mods
p_load(sjmisc) 
p_load(effects)
p_load(sjstats) #use for r2 functions


sjPlot::plot_model(mod_lme4)
`geom_smooth()` using formula 'y ~ x'
`geom_smooth()` using formula 'y ~ x'

sjPlot:: tab_model(mod_lme4)
effects_costp <- effects::effect(term= "cost_p", mod= mod_lme4) %>% as.data.frame()
summary(effects_costp) #output of what the values are
     cost_p           fit             se            lower          upper     
 Min.   : 45.0   Min.   :2.17   Min.   :0.142   Min.   :1.87   Min.   :2.46  
 1st Qu.: 66.0   1st Qu.:3.18   1st Qu.:0.208   1st Qu.:2.75   1st Qu.:3.61  
 Median : 88.0   Median :4.24   Median :0.278   Median :3.66   Median :4.81  
 Mean   : 87.8   Mean   :4.23   Mean   :0.277   Mean   :3.65   Mean   :4.80  
 3rd Qu.:110.0   3rd Qu.:5.30   3rd Qu.:0.347   3rd Qu.:4.58   3rd Qu.:6.02  
 Max.   :130.0   Max.   :6.26   Max.   :0.410   Max.   :5.41   Max.   :7.11  
  ggplot() + 
  #2
  geom_point(data=df_SMD[[i]], aes(cost_p, log(absolute_lift))) + 
  #3
  geom_point(data=effects_costp, aes(x=cost_p, y=fit), color="blue") +
  #4
  geom_line(data=effects_costp, aes(x=cost_p, y=fit), color="blue") +
  #5
  geom_ribbon(data= effects_costp, aes(x=cost_p, ymin=lower, ymax=upper), alpha= 0.3, fill="blue") +
  #6
  labs(x="cost_p", y="Log(Absolute Lift)")

LS0tDQp0aXRsZTogIjAzX0NMU19TcGVuZF9SZXNwb25zZV9DdXJ2ZXNfTm9fUG9seSINCmF1dGhvcjogIkVzc2VuY2UgR2xvYmFsIEFkdmFuY2VkIEFuYWx5dGljcyBUZWFtIg0KZGF0ZTogImByIFN5cy5EYXRlKClgIg0Kb3V0cHV0Og0KICBodG1sX25vdGVib29rOg0KICAgIHRvYzogeWVzDQogICAgdG9jX2Zsb2F0OiB5ZXMNCiAgICBudW1iZXJfc2VjdGlvbnM6IG5vDQogICAgdGhlbWU6IGNlcnVsZWFuDQogICAgaGlnaGxpZ2h0OiB6ZW5idXJuDQogICAgZGZfcHJpbnQ6IHBhZ2VkDQotLS0NCg0KYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9DQprbml0cjo6b3B0c19jaHVuayRzZXQoZWNobyA9IFRSVUUpDQpvcHRpb25zKGtuaXRyLnRhYmxlLmZvcm1hdCA9ICJodG1sIikNCm9wdGlvbnMoZGlnaXRzID0gNSkNCm9wdGlvbnMoc2NpcGVuID0gMTAwKQ0Ka25pdHI6Om9wdHNfY2h1bmskc2V0KHRpZHkub3B0cyA9IGxpc3Qod2lkdGguY3V0b2ZmID0gODApLCB0aWR5ID0gVFJVRSkNCmtuaXRyOjpvcHRzX2NodW5rJHNldChmaWcud2lkdGggPSAxNSkNCmtuaXRyOjpvcHRzX2NodW5rJHNldChmaWcuaGVpZ2h0ID0gMTApDQojIGluc3RhbGwucGFja2FnZXMoInBhY21hbiIpDQpsaWJyYXJ5KHBhY21hbikgIyBmb3IgcXVpY2sgbG9hZC9pbnN0YWxsIG9mIHBhY2thZ2VzDQpwX2xvYWQoZHBseXIsIHJlYWRyLCB0aWR5dmVyc2UsIHJldGljdWxhdGUsIGx1YnJpZGF0ZSwgamFuaXRvciwgc3FsZGYsIGdvb2dsZXNoZWV0czQpDQpwX2xvYWQoc2tpbXIsIHNwbGl0c3RhY2tzaGFwZSwgc3RyaW5nciwgcnFkYXRhdGFibGUpDQpwX2xvYWQobW9tZW50cykNCnBfbG9hZChrYWJsZUV4dHJhKQ0KcF9sb2FkKGdncGxvdDIsIHBsb3RseSwgZWNoYXJ0czRyLCBnZ3B1YnIsIGZvcmNhdHMsIHNjYWxlcywgUkNvbG9yQnJld2VyLGdyaWRFeHRyYSkNCnBfbG9hZChnZ3RoZW1lcykNCnBfbG9hZChjYXJldCwgcmVjaXBlcykNCnBfbG9hZChnbG1uZXQpDQpwX2xvYWQoZWxhc3RpY25ldCkNCnBfbG9hZChNZXRyaWNzKQ0KcF9sb2FkKGZhc3REdW1taWVzKQ0KcF9sb2FkKGJyb29tKQ0KcF9sb2FkKGh0bWx3aWRnZXRzKQ0KcF9sb2FkKHNvbGl0dWRlKQ0KcF9sb2FkKG1sYmVuY2gpDQpwX2xvYWQodXdvdCkNCnBfbG9hZChsbWU0KQ0KcF9sb2FkKGxtZXJUZXN0KQ0KYGBgDQoNCiMgVXNlIERhdGFzZXQgY3JlYXRlZCBmcm9tIDAyX0NMU19EYXRhX1N1bW1hcnlfMjAyMl8wOTE0X0RhdGFfQW5hbHlzaXMgRmlsZQ0KDQojIyBMb2FkaW5nIERhdGENCg0KIyMjIExvYWQgR29vZ2xlIFNoZWV0DQoNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGUgPC0gcmVhZF9zaGVldCgNCiAgImh0dHBzOi8vZG9jcy5nb29nbGUuY29tL3NwcmVhZHNoZWV0cy9kLzFONDhyVGVxN21kMHY4dzhwR184WElpdWFwUEhRQWVPNVdvV0lCM2VhY2VJL2VkaXQjZ2lkPTE0NDkzNTEzNzciLA0KICBzaGVldCA9ICJGaW5hbERhdGFzZXRfMjAyMl9VcGRhdGUiDQopICU+JQ0KICBtdXRhdGUoDQogICAgU2lnbmlmaWNhbnRfU3BlbmQgPQ0KICAgICAgYXMubnVtZXJpYygNCiAgICAgICAgY2FzZV93aGVuKA0KICAgICAgICAgIHByb2JhYmlsaXR5X29mX2xpZnQgPj0gMC45IH4gMSwNCiAgICAgICAgICBUUlVFIH4gMA0KICAgICAgICApDQogICAgICApLA0KICAgIGNvdW50cnkgPSBjYXNlX3doZW4oDQogICAgICBjb3VudHJ5ID09ICJOQSIgfiAiVVMiLA0KICAgICAgVFJVRSB+IGNvdW50cnkNCiAgICApLA0KICAgIHJlZ2lvbl92MiA9IGNhc2Vfd2hlbigNCiAgICAgIGNvdW50cnkgPT0gIlVTIiB+ICJOQSIsDQogICAgICBjb3VudHJ5ID09ICJDQSIgfiAiTkEiLA0KICAgICAgY291bnRyeSA9PSAiVVMgKyBDQSIgfiAiTkEiLA0KICAgICAgVFJVRSB+IHJlZ2lvbg0KICAgICkNCiAgKSAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgIT0gIlNlYXJjaCIpICU+JQ0KICAjIGZpbHRlciBvdXQgc3R1ZGllcyB3aXRob3V0IHJlcG9ydGVkIGxpZnRzDQogIGZpbHRlcihleHBvc2VkICE9IC0xKSAlPiUNCiAgIyBmaWx0ZXIgb3V0IGdvb2dsZSBwYXkgc3R1ZHkNCiAgZmlsdGVyKHN0dWR5X2lkICE9ICIxNDkxNDIyMTciKSAlPiUNCiAgIyBmaWx0ZXIgb3V0IHZlcnkgbmVnYXRpdmUgYWJzb2x1dGUgbGlmdHMNCiAgZmlsdGVyKGFic29sdXRlX2xpZnQgPiAtMTAwMCkgJT4lDQogIG11dGF0ZSgNCiAgICBwYSA9IGNhc2Vfd2hlbigNCiAgICAgIHBhID09ICJHb29nbGUgQWRzIiB+ICJTTUIiLCAjIFN0ZXAgMQ0KICAgICAgcGEgPT0gIllvdVR1YmUiICYgY29udmVyc2lvbiAhPSAiVHlwZSAyNTY1MjI5NDIgKFtNQ0NdIFlvdVR1YmUgVFYgLSBXZWIgLSBUcmlhbCBTdGFydCkiIH4gIllUTVAiLCAjIFN0ZXAgMg0KICAgICAgcGEgPT0gIllvdVR1YmUgUHJlbWl1bSIgfiAiWVRNUCIsICMgU3RlcCAyDQogICAgICBjb252ZXJzaW9uID09ICJUeXBlIDI1NjUyMjk0MiAoW01DQ10gWW91VHViZSBUViAtIFdlYiAtIFRyaWFsIFN0YXJ0KSIgfiAiWW91VHViZSBUViIsICMgU3RlcCAyDQogICAgICBwYSA9PSAiQ2xvdWQiICYgY29udmVyc2lvbiAhPSAiVHlwZSAxNDI1NzgwMyAoRW50ZXJwcmlzZSAtIEFwcHMgLSBTaWdudXAgQ29uZmlybSAtIFVuaXF1ZSkiIH4gIkNsb3VkIFdvcmtzcGFjZSIsICMgU3RlcCAzDQogICAgICBwYSA9PSAiQ2xvdWQiICYgY29udmVyc2lvbiA9PSAiVHlwZSAxNDI1NzgwMyAoRW50ZXJwcmlzZSAtIEFwcHMgLSBTaWdudXAgQ29uZmlybSAtIFVuaXF1ZSkiIH4gIkNsb3VkIEdDUCIsICMgU3RlcCAzDQogICAgICBwYSA9PSAiUHJvamVjdCBGaSIgfiAiR29vZ2xlIEZpIiwgIyBTdGVwIDQNCiAgICAgIHBhID09ICJHb29nbGUgQ2hyb21lIiB+ICJDaHJvbWUiLA0KICAgICAgVFJVRSB+IHBhDQogICAgKQ0KICApICU+JQ0KICBtdXRhdGUoDQogICAgcGFyc2VkX3R5cGUgPSBwYXJzZV9udW1iZXIoY29udmVyc2lvbiksDQogICAgZ3JvdXBlZF9jb252ZXJzaW9uID0gY2FzZV93aGVuKA0KICAgICAgY29udmVyc2lvbiAlaW4lIGMoIkNocm9tZWJvb2sgTWljcm9zaXRlIFJlZmVycmFsIENsaWNrcyBRNCAyMDE1IiwgIlR5cGUgMjUxNDIyNzI5IChDaHJvbWVib29rcyBNaWNyb3NpdGUgUmVmZXJyYWwgQ2xpY2tzIChRNCAyMDE3KSkiKSB+ICJDaHJvbWVib29rIFJlZmVycmFscyIsDQogICAgICBjb252ZXJzaW9uICVpbiUgYygiRGVza3RvcCBEb3dubG9hZHMiLCAiVHlwZSAxMTU0MTU0NyAoRGVza3RvcCBEb3dubG9hZCkiKSB+DQogICAgICAgICJEZXNrdG9wIERvd25sb2FkcyIsDQogICAgICBwYSA9PSAiUGl4ZWwiIH4gIk1vYmlsZSBDb252ZXJzaW9ucyIsDQogICAgICBwYSA9PSAiRFNNIiB+ICJOb24tTW9iaWxlIERldmljZSBDb252ZXJzaW9ucyIsDQogICAgICBjb252ZXJzaW9uID09ICJUeXBlIDMwMjk4Mjk1NCAoTGVuYSAtIFAgTGVhZCkiIH4gIkxlbmEgUCBMZWFkIiwNCiAgICAgIGNvbnZlcnNpb24gPT0gIlR5cGUgMjg4MzQ3MDA4IChMRU5BIC0gQiBMZWFkKSIgfiAiTGVuYSBCIExlYWQiLA0KICAgICAgY29udmVyc2lvbiA9PSAiVHlwZSAyODg2OTc2NTMgKExFTkEgLSBRIExlYWQpIiB+ICJMZW5hIFEgTGVhZCIsDQogICAgICBwYXJzZWRfdHlwZSAlaW4lIGMoMTgxMjgzOTkzLCA4NTU1MDg2ODYpIH4gIldvcmtzcGFjZSBGcmVlIFRyaWFsIFN0YXJ0IiwNCiAgICAgIHBhcnNlZF90eXBlID09IDMzMDc1NTY0MSB+ICJNaWNyb3NpdGUgQ29udmVyc2lvbnMiLA0KICAgICAgcGFyc2VkX3R5cGUgPT0gMTQyNTc4MDMgfiAiRW50ZXJwcmlzZSBTaWdudXBzIiwNCiAgICAgIHBhcnNlZF90eXBlID09IDI4OTY4MDcxMiB+ICJHb29nbGUoaU9zKSBGaXJzdCBPcGVuIiwNCiAgICAgIHBhcnNlZF90eXBlID09IDI1NjUyMjk0MiB+ICJZb3VUdWJlIFRWIC0gV2ViIC0gVHJpYWwgU3RhcnQiLA0KICAgICAgcGFyc2VkX3R5cGUgJWluJSBjKDQ1MjM5MTUzNCwgMjIxNDk3ODMzLCAyNzcxNTAwNzQpIH4gIlRyaWFsIFNpZ251cHMgQ29tcGxldGUiLA0KICAgICAgVFJVRSB+IGNvbnZlcnNpb24NCiAgICApLA0KICAgIHBhID0gY2FzZV93aGVuKA0KICAgICAgY29udmVyc2lvbiA9PSAiVHlwZSAyODg2OTc2NTMgKExFTkEgLSBRIExlYWQpIiB+ICJTTUItUUxlYWQiLA0KICAgICAgVFJVRSB+IHBhDQogICAgKQ0KICApICU+JQ0KICBmaWx0ZXIoYWJzb2x1dGVfbGlmdCA+IDApDQoNCg0KIyBhbGwuZXF1YWwoRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGUsRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX3YzKQ0KYGBgDQoNCiMgQ3JlYXRlIEFsbCBSZXNwb25zZSBDdXJ2ZXMgb25seSBub3JtYWwgcG93ZXJzDQoNCiMjIEZvbGRlciBmb3IgYWxsIE91dHB1dCBhbmQgc2NyaXB0cw0KDQpgYGB7cn0NCmZvbGRlcl9uYW1lIDwtIHBhc3RlMCgiT3V0cHV0LyIsICJvdXRwdXRmaWxlc18iLCBTeXMuRGF0ZSgpLCAiXyIsICJSdW4xIiwgIi8iKQ0KZGlyLmNyZWF0ZShmb2xkZXJfbmFtZSkgIyBpdCB3aWxsIHRocm93IGEgd2FybmluZyBpZiBmb2xkZXIgZXhpc3RzDQoNCiMgZmlsZS5zb3VyY2VzMiA8LSBsaXN0LmZpbGVzKHBhdGggPSAiT3V0cHV0L291dHB1dGZpbGVzXzIwMjItMTAtMTRfUnVuMS8vIiwgcGF0dGVybiA9Ii5odG1sfC5wbmciLCBmdWxsLm5hbWVzID0gVFJVRSkNCmZpbGUuc291cmNlcyA8LSBsaXN0LmZpbGVzKHBhdGggPSAiUlNjcmlwdHMvIiwgcGF0dGVybiA9ICIqLlIiLCBmdWxsLm5hbWVzID0gVFJVRSkNCnNhcHBseShmaWxlLnNvdXJjZXMsIHNvdXJjZSwgLkdsb2JhbEVudikNCmBgYA0KDQojIyBDaGVjayBwYXJhbWV0ZXJzDQoNCmBgYHtyfQ0KDQojIyMgcG93ZXJzIHRvIHRyeQ0KcG93ZXJzIDwtIHNlcSgwLjEsIDAuOSwgYnkgPSAwLjAxKQ0KcG93ZXJzMiA8LSAxDQoNCiMjIyBQb3dlcnMgdG8gVHJ5DQojcG93ZXJzIDwtIHNlcSgwLjEsIDAuOSwgYnkgPSAwLjAxKQ0KI3Bvd2VyczIgPC1zZXEoMS41LDMsIGJ5ID0gMC4yNSkNCg0KDQojIyMgTGFtYmRhIHBhcmFtZXRlcnMNCnBhcmFtZXRlcnMgPC0gYygNCiAgIyAgc2VxKDAuMSwgMiwgYnkgPTAuMSkgLCAgc2VxKDIsIDUsIDAuNSkgLA0KICBzZXEoNSwgMjksIDEpDQogICxzZXEoMzAsIDEwMiwgNCkNCiAgLHNlcSgxMTAsIDEwMDAsIDE1KQ0KICAsc2VxKDEwMDAsIDEwMDIwLCA1MDApDQopDQoNCiMjIyBlbGFzdGljbmV0IHBhcmFtZXRlcnMNCmFscGhhX3BhcmFtZXRlcnMgPC0gYyhzZXEoMCwgMSwgMC4yNSkpDQoNCiMgRm9yIFRlc3RpbmcgUHVycG9zZXMNCiNhbHBoYV9wYXJhbWV0ZXJzIDwtIGMoc2VxKDEsIDEsIDEpKQ0KDQpgYGANCg0KIyMgVGVzdGluZyBEaWZmZXJlbnQgTW9kZWwgVHlwZXMNCg0KIyMjIENocm9tZQ0KDQojIyMjIERhdGEgUmVhZGluDQoNCmBgYHtyfQ0KDQpzdGFydF90aW1lIDwtIFN5cy50aW1lKCkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2hyb21lX3ByZSA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZSAlPiUNCiAgZmlsdGVyKHBhID09ICJDaHJvbWUiKSAlPiUNCiAgbXV0YXRlKA0KICAgIGlkMiA9IHJvd19udW1iZXIoKQ0KICApDQoNCmRmX3Rlc3QgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2hyb21lX3ByZSAlPiUNCiAgc2VsZWN0KA0KICAgIHJlZ2lvbl92MiwgY291bnRyeSwgY2hhbm5lbCwgdGFjdGljLA0KICAgIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cDphYnNvbHV0ZV9saWZ0DQogICkNCg0KaXNvX2Nocm9tZSA8LSBpc29sYXRpb25Gb3Jlc3QkbmV3KHNhbXBsZV9zaXplID0gbnJvdyhkZl90ZXN0KSwgbnVtX3RyZWVzID0gMTAwMDAsIHNlZWQgPSAxMTUzKQ0KDQppc29fY2hyb21lJGZpdChkZl90ZXN0KQ0KDQpzY29yZXNfdHJhaW4gPC0gZGZfdGVzdCAlPiUNCiAgaXNvX2Nocm9tZSRwcmVkaWN0KCkgJT4lDQogIGFycmFuZ2UoZGVzYyhhbm9tYWx5X3Njb3JlKSkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2hyb21lX3ByZTIgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2hyb21lX3ByZSAlPiUNCiAgbGVmdF9qb2luKHNjb3Jlc190cmFpbiwgYnkgPSBjKCJpZDIiID0gImlkIikpICU+JSANCiAgZmlsdGVyKGF2ZXJhZ2VfZGVwdGggPiAzKQ0KDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2Nocm9tZSA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9jaHJvbWVfcHJlMiAlPiUNCiAgbmFtZWRfZ3JvdXBfc3BsaXQodGFjdGljKQ0KYGBgDQoNCiMjIyMgUnVuIE1vZGVsDQoNCmBgYHtyLCB3YXJuaW5nID0gZmFsc2V9DQoNCg0KZml0c19ub25fc2VhcmNoX2Nocm9tZSA8LSBtb2RlbF93cmFwcGVyX2Z1bmN0aW9uKGRmID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2hyb21lLHBvbHlfaW5kID0gMCkNCg0KYmVzdF9pbmRfbm9uX3NlYXJjaF9jaHJvbWUgPC0gDQogIGxhcHBseSgxOmxlbmd0aChGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9jaHJvbWUpLCBiZXN0X2luZF9mdW5jdGlvbixkZiA9IGZpdHNfbm9uX3NlYXJjaF9jaHJvbWUsDQogICAgICAgICBkZjIgPSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9jaHJvbWUpIA0KDQpjb2VmX25vbl9zZWFyY2hfY2hyb21lIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfY2hyb21lICU+JSBiaW5kX3Jvd3MgI21ha2UgYSBtYXRyaXggb2YgYWxsIGNvZWZzDQoNCmJlc3RfZml0X25vbl9zZWFyY2hfY2hyb21lIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfY2hyb21lICU+JQ0KICBzZXRfbmFtZXMobmFtZXNfZnVuY3Rpb24oRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2hyb21lX3ByZSwgdGFjdGljKSkgIA0KYGBgDQoNCiMjIyMgQ3JlYXRlIEdyYXBoIE9iamVjdA0KDQpgYGB7cn0NCmdyYXBoX2xpc3RfY2hyb21lIDwtIGxhcHBseSgxOmxlbmd0aChiZXN0X2ZpdF9ub25fc2VhcmNoX2Nocm9tZSksIGdyYXBoaW5nX2Z1bmN0aW9uNCwgZGYxID0gYmVzdF9maXRfbm9uX3NlYXJjaF9jaHJvbWUsIGRmMiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2Nocm9tZSkNCmBgYA0KDQpgYGB7cn0NCmVuZF90aW1lIDwtIFN5cy50aW1lKCkNCg0KdGltZV9jaHJvbWUgPSBlbmRfdGltZSAtIHN0YXJ0X3RpbWUNCg0KdGltZV9jaHJvbWUNCmBgYA0KDQojIyMgQ2xvdWQNCg0KIyMjIyBEYXRhIFJlYWRpbg0KDQpgYGB7cn0NCnN0YXJ0X3RpbWUgPC0gU3lzLnRpbWUoKQ0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9jbG91ZF9wcmUgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGUgJT4lDQogIGZpbHRlcihwYSAlaW4lIGMoIkNsb3VkIEdDUCIsICJDbG91ZCBXb3Jrc3BhY2UiKSkgJT4lDQogIG11dGF0ZSgNCiAgICBwYSA9ICJDbG91ZCIsDQogICAgcGEyID0gIkNsb3VkIC0gQWxsIENoYW5uZWwiDQogICkgJT4lDQogIG11dGF0ZSgNCiAgICBpZDIgPSByb3dfbnVtYmVyKCkNCiAgKQ0KDQpkZl90ZXN0IDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2Nsb3VkX3ByZSAlPiUNCiAgIyBzZWxlY3QoLXN0dWR5X2lkLCAtaWQyLCAtcmVnaW9uLCAtc2NhbGluZ19mYWN0b3IsIC1xdWFydGVyLCAtcGEsIC1zdHVkeV9uYW1lKQ0KICBzZWxlY3QoDQogICAgcmVnaW9uX3YyLCBjb3VudHJ5LCBjaGFubmVsLCB0YWN0aWMsDQogICAgIyB0cmVhdG1lbnRfdXNlcl9jb3VudDpjb250cm9sLA0KICAgIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cDphYnNvbHV0ZV9saWZ0LCBwYXJzZWRfdHlwZQ0KICApDQoNCmlzb19jbG91ZCA8LSBpc29sYXRpb25Gb3Jlc3QkbmV3KHNhbXBsZV9zaXplID0gbnJvdyhkZl90ZXN0KSwgbnVtX3RyZWVzID0gMTAwMDAsIHNlZWQgPSAxMTUzKQ0KDQppc29fY2xvdWQkZml0KGRmX3Rlc3QpDQoNCnNjb3Jlc190cmFpbiA8LSBkZl90ZXN0ICU+JQ0KICBpc29fY2xvdWQkcHJlZGljdCgpICU+JQ0KICBhcnJhbmdlKGRlc2MoYW5vbWFseV9zY29yZSkpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2Nsb3VkX3ByZTIgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2xvdWRfcHJlICU+JQ0KICBsZWZ0X2pvaW4oc2NvcmVzX3RyYWluLCBieSA9IGMoImlkMiIgPSAiaWQiKSkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2xvdWQgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2xvdWRfcHJlMiAlPiUNCiAgbmFtZWRfZ3JvdXBfc3BsaXQocGEyKQ0KDQpgYGANCg0KIyMjIyBSdW4gTW9kZWwNCg0KYGBge3IsIHdhcm5pbmcgPSBmYWxzZX0NCmZpdHNfbm9uX3NlYXJjaF9jbG91ZCA8LSBtb2RlbF93cmFwcGVyX2Z1bmN0aW9uKGRmID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2xvdWQscG9seV9pbmQgPSAwKQ0KDQpiZXN0X2luZF9ub25fc2VhcmNoX2Nsb3VkIDwtIA0KICBsYXBwbHkoMTpsZW5ndGgoRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2xvdWQpLCBiZXN0X2luZF9mdW5jdGlvbixkZiA9IGZpdHNfbm9uX3NlYXJjaF9jbG91ZCwNCiAgICAgICAgIGRmMiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2Nsb3VkKSANCg0KY29lZl9ub25fc2VhcmNoX2Nsb3VkIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfY2xvdWQgJT4lIGJpbmRfcm93cyAjbWFrZSBhIG1hdHJpeCBvZiBhbGwgY29lZnMNCg0KYmVzdF9maXRfbm9uX3NlYXJjaF9jbG91ZCA8LSBiZXN0X2luZF9ub25fc2VhcmNoX2Nsb3VkICU+JQ0KICBzZXRfbmFtZXMobmFtZXNfZnVuY3Rpb24oRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2xvdWRfcHJlLCBwYTIpKSAgDQpgYGANCg0KIyMjIyBDcmVhdGUgR3JhcGggT2JqZWN0DQoNCmBgYHtyfQ0KZ3JhcGhfbGlzdF9jbG91ZCA8LSBsYXBwbHkoMTpsZW5ndGgoYmVzdF9maXRfbm9uX3NlYXJjaF9jbG91ZCksIGdyYXBoaW5nX2Z1bmN0aW9uNCwgZGYxID0gYmVzdF9maXRfbm9uX3NlYXJjaF9jbG91ZCwgZGYyID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfY2xvdWQpDQpgYGANCg0KDQpgYGB7cn0NCmVuZF90aW1lIDwtIFN5cy50aW1lKCkNCg0KdGltZV9jbG91ZCA9IGVuZF90aW1lIC0gc3RhcnRfdGltZQ0KYGBgDQoNCiMjIyBZb3VUdWJlDQoNCiMjIyMgRGF0YSBSZWFkaW4NCg0KYGBge3J9DQoNCnN0YXJ0X3RpbWUgPC0gU3lzLnRpbWUoKQ0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV95b3V0dWJlX3ByZSA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZSAlPiUNCiAgZmlsdGVyKHBhICVpbiUgYygiWW91VHViZSBUViIsICJZVE1QIikpICU+JQ0KICBtdXRhdGUoDQogICAgcGEgPSAiWW91VHViZSIsDQogICAgcGEyID0gIllvdVR1YmUiDQogICkgJT4lDQogICMgIGZpbHRlcihhYnNvbHV0ZV9saWZ0IDwgNTAwMCkgJT4lDQogIG11dGF0ZSgNCiAgICBpZDIgPSByb3dfbnVtYmVyKCkNCiAgKQ0KDQpkZl90ZXN0IDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3lvdXR1YmVfcHJlICU+JQ0KICAjIHNlbGVjdCgtc3R1ZHlfaWQsIC1pZDIsIC1yZWdpb24sIC1zY2FsaW5nX2ZhY3RvciwgLXF1YXJ0ZXIsIC1wYSwgLXN0dWR5X25hbWUpDQogIHNlbGVjdCgNCiAgICByZWdpb25fdjIsIGNvdW50cnksIGNoYW5uZWwsIHRhY3RpYywNCiAgICAjIHRyZWF0bWVudF91c2VyX2NvdW50OmNvbnRyb2wsDQogICAgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwOmFic29sdXRlX2xpZnQsIHBhcnNlZF90eXBlDQogICkNCg0KaXNvX3l0IDwtIGlzb2xhdGlvbkZvcmVzdCRuZXcoc2FtcGxlX3NpemUgPSBucm93KGRmX3Rlc3QpLCBudW1fdHJlZXMgPSAxMDAwMCwgc2VlZCA9IDExNTMpDQoNCmlzb195dCRmaXQoZGZfdGVzdCkNCg0Kc2NvcmVzX3RyYWluIDwtIGRmX3Rlc3QgJT4lDQogIGlzb195dCRwcmVkaWN0KCkgJT4lDQogIGFycmFuZ2UoZGVzYyhhbm9tYWx5X3Njb3JlKSkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfeW91dHViZV9wcmUyIDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3lvdXR1YmVfcHJlICU+JQ0KICBsZWZ0X2pvaW4oc2NvcmVzX3RyYWluLCBieSA9IGMoImlkMiIgPSAiaWQiKSkgJT4lDQogIGZpbHRlcihhdmVyYWdlX2RlcHRoID4gMy44OSkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfeW91dHViZSA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV95b3V0dWJlX3ByZTIgJT4lDQogIG5hbWVkX2dyb3VwX3NwbGl0KHJlZ2lvbl92MikNCmBgYA0KDQojIyMjIFJ1biBNb2RlbA0KDQpgYGB7ciwgd2FybmluZyA9IGZhbHNlfQ0KZml0c19ub25fc2VhcmNoX3lvdXR1YmUgPC0gbW9kZWxfd3JhcHBlcl9mdW5jdGlvbihkZiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3lvdXR1YmUscG9seV9pbmQgPSAwKQ0KDQpiZXN0X2luZF9ub25fc2VhcmNoX3lvdXR1YmUgPC0gDQogIGxhcHBseSgxOmxlbmd0aChGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV95b3V0dWJlKSwgYmVzdF9pbmRfZnVuY3Rpb24sZGYgPSBmaXRzX25vbl9zZWFyY2hfeW91dHViZSwNCiAgICAgICAgIGRmMiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3lvdXR1YmUpIA0KDQpjb2VmX25vbl9zZWFyY2hfeW91dHViZSA8LSBiZXN0X2luZF9ub25fc2VhcmNoX3lvdXR1YmUgJT4lIGJpbmRfcm93cyAjbWFrZSBhIG1hdHJpeCBvZiBhbGwgY29lZnMNCg0KYmVzdF9maXRfbm9uX3NlYXJjaF95b3V0dWJlIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfeW91dHViZSAlPiUNCiAgc2V0X25hbWVzKG5hbWVzX2Z1bmN0aW9uKEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3lvdXR1YmVfcHJlLCBwYTIpKSAgDQpgYGANCg0KIyMjIyBDcmVhdGUgR3JhcGggT2JqZWN0DQoNCmBgYHtyfQ0KZ3JhcGhfbGlzdF95b3V0dWJlIDwtIGxhcHBseSgxOmxlbmd0aChiZXN0X2ZpdF9ub25fc2VhcmNoX3lvdXR1YmUpLCBncmFwaGluZ19mdW5jdGlvbjQsIGRmMSA9IGJlc3RfZml0X25vbl9zZWFyY2hfeW91dHViZSwgZGYyID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfeW91dHViZSkNCmBgYA0KDQoNCmBgYHtyfQ0KZW5kX3RpbWUgPC0gU3lzLnRpbWUoKQ0KDQp0aW1lX3lvdXR1YmUgPSBlbmRfdGltZSAtIHN0YXJ0X3RpbWUNCmBgYA0KDQoNCiMjIyBEU00NCg0KIyMjIyBEYXRhIFJlYWRpbg0KDQpgYGB7cn0NCg0Kc3RhcnRfdGltZSA8LSBTeXMudGltZSgpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2RzbV9wcmUgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGUgJT4lDQogIGZpbHRlcihwYSA9PSAiRFNNIikgJT4lDQogIGZpbHRlcihyZWdpb25fdjIgIT0gIkFQQUMiKSAlPiUNCiAgIyBmaWx0ZXIoYWJzb2x1dGVfbGlmdCA8IDEwMDApICMgJT4lDQogICMgZmlsdGVyKHN0dWR5X2lkICE9ICc2Mjk3NDIwJykgIyU+JQ0KICAjICBmaWx0ZXIoc3R1ZHlfaWQgIT0nMTQ5MTYxNzExJykgJT4lDQogICMgIGZpbHRlcihzdHVkeV9pZCAhPSAnMTQ4NjEzMDAyJykgJT4lDQogICMgZmlsdGVyKHN0dWR5X2lkICE9JzMyODQ2MjUnKSAlPiUNCiAgIyAgZmlsdGVyKHN0dWR5X2lkICE9JzMzMjkxMzEnKQ0KICBtdXRhdGUoDQogICAgaWQyID0gcm93X251bWJlcigpDQogICkNCg0KZGZfdGVzdCA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9kc21fcHJlICU+JQ0KICAjIHNlbGVjdCgtc3R1ZHlfaWQsIC1pZDIsIC1yZWdpb24sIC1zY2FsaW5nX2ZhY3RvciwgLXF1YXJ0ZXIsIC1wYSwgLXN0dWR5X25hbWUpDQogIHNlbGVjdCgNCiAgICByZWdpb25fdjIsIGNvdW50cnksIGNoYW5uZWwsIHRhY3RpYywNCiAgICAjIHRyZWF0bWVudF91c2VyX2NvdW50OmNvbnRyb2wsDQogICAgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwOmFic29sdXRlX2xpZnQNCiAgKQ0KDQppc29fZHNtIDwtIGlzb2xhdGlvbkZvcmVzdCRuZXcoc2FtcGxlX3NpemUgPSBucm93KGRmX3Rlc3QpLCBudW1fdHJlZXMgPSAxMDAwMCwgc2VlZCA9IDExNTIpDQoNCmlzb19kc20kZml0KGRmX3Rlc3QpDQoNCnNjb3Jlc190cmFpbiA8LSBkZl90ZXN0ICU+JQ0KICBpc29fZHNtJHByZWRpY3QoKSAlPiUNCiAgYXJyYW5nZShkZXNjKGFub21hbHlfc2NvcmUpKQ0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9kc21fcHJlMiA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9kc21fcHJlICU+JQ0KICBsZWZ0X2pvaW4oc2NvcmVzX3RyYWluLCBieSA9IGMoImlkMiIgPSAiaWQiKSkgJT4lDQogIGZpbHRlcihhdmVyYWdlX2RlcHRoID4gNSkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtIDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2RzbV9wcmUyICU+JQ0KICBuYW1lZF9ncm91cF9zcGxpdChyZWdpb25fdjIsIGNoYW5uZWwpDQpgYGANCg0KIyMjIyBSdW4gTW9kZWwNCg0KYGBge3IsIHdhcm5pbmcgPSBmYWxzZX0NCmZpdHNfbm9uX3NlYXJjaF9kc20gPC0gbW9kZWxfd3JhcHBlcl9mdW5jdGlvbihkZiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2RzbSxwb2x5X2luZCA9IDApDQoNCmJlc3RfaW5kX25vbl9zZWFyY2hfZHNtIDwtIA0KICBsYXBwbHkoMTpsZW5ndGgoRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtKSwgYmVzdF9pbmRfZnVuY3Rpb24sZGYgPSBmaXRzX25vbl9zZWFyY2hfZHNtLA0KICAgICAgICAgZGYyID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtKQ0KDQpjb2VmX25vbl9zZWFyY2hfZHNtIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfZHNtICU+JSBiaW5kX3Jvd3MgI21ha2UgYSBtYXRyaXggb2YgYWxsIGNvZWZzDQoNCmJlc3RfZml0X25vbl9zZWFyY2hfZHNtIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfZHNtICU+JQ0KICBzZXRfbmFtZXMobmFtZXNfZnVuY3Rpb24oRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtX3ByZSwgcmVnaW9uX3YyLGNoYW5uZWwpKSAgDQpgYGANCg0KIyMjIyBDcmVhdGUgR3JhcGggT2JqZWN0DQoNCmBgYHtyfQ0KZ3JhcGhfbGlzdF9kc20gPC0gbGFwcGx5KDE6bGVuZ3RoKGJlc3RfZml0X25vbl9zZWFyY2hfZHNtKSwgZ3JhcGhpbmdfZnVuY3Rpb240LCBkZjEgPSBiZXN0X2ZpdF9ub25fc2VhcmNoX2RzbSwgZGYyID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtKQ0KYGBgDQoNCg0KYGBge3J9DQplbmRfdGltZSA8LSBTeXMudGltZSgpDQoNCnRpbWVfZHNtID0gZW5kX3RpbWUgLSBzdGFydF90aW1lDQpgYGANCg0KDQojIyMgUGl4ZWwNCg0KIyMjIyBEYXRhIFJlYWRpbg0KDQpgYGB7cn0NCg0Kc3RhcnRfdGltZSA8LSBTeXMudGltZSgpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3BpeGVsX3ByZSA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZSAlPiUNCiAgZmlsdGVyKHBhID09ICJQaXhlbCIpICU+JQ0KICBtdXRhdGUoDQogICAgcGEyID0gIlBpeGVsIC0gQWxsIENoYW5uZWwiDQogICkgJT4lDQogICMgICBmaWx0ZXIoYWJzb2x1dGVfbGlmdCA8IDEwMDApICAlPiUNCiAgIyBmaWx0ZXIoc3R1ZHlfaWQgIT0gJzYyOTc0MjAnKSAjJT4lDQogICMgIGZpbHRlcihzdHVkeV9pZCAhPScxNDkxNjE3MTEnKSAlPiUNCiAgIyAgZmlsdGVyKHN0dWR5X2lkICE9ICcxNDg2MTMwMDInKSAlPiUNCiAgIyBmaWx0ZXIoc3R1ZHlfaWQgIT0nMzI4NDYyNScpICU+JQ0KICAjICBmaWx0ZXIoc3R1ZHlfaWQgIT0nMzMyOTEzMScpDQogIG11dGF0ZSgNCiAgICBpZDIgPSByb3dfbnVtYmVyKCkNCiAgKQ0KDQpkZl90ZXN0IDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3BpeGVsX3ByZSAlPiUNCiAgIyBzZWxlY3QoLXN0dWR5X2lkLCAtaWQyLCAtcmVnaW9uLCAtc2NhbGluZ19mYWN0b3IsIC1xdWFydGVyLCAtcGEsIC1zdHVkeV9uYW1lKQ0KICBzZWxlY3QoDQogICAgcmVnaW9uX3YyLCBjb3VudHJ5LCBjaGFubmVsLCB0YWN0aWMsDQogICAgIyB0cmVhdG1lbnRfdXNlcl9jb3VudDpjb250cm9sLA0KICAgIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cDphYnNvbHV0ZV9saWZ0DQogICkNCg0KaXNvX3BpeGVsIDwtIGlzb2xhdGlvbkZvcmVzdCRuZXcoc2FtcGxlX3NpemUgPSBucm93KGRmX3Rlc3QpLCBudW1fdHJlZXMgPSAxMDAwMCwgc2VlZCA9IDExNTIpDQoNCmlzb19waXhlbCRmaXQoZGZfdGVzdCkNCg0Kc2NvcmVzX3RyYWluIDwtIGRmX3Rlc3QgJT4lDQogIGlzb19waXhlbCRwcmVkaWN0KCkgJT4lDQogIGFycmFuZ2UoZGVzYyhhbm9tYWx5X3Njb3JlKSkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfcGl4ZWxfcHJlMiA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9waXhlbF9wcmUgJT4lDQogIGxlZnRfam9pbihzY29yZXNfdHJhaW4sIGJ5ID0gYygiaWQyIiA9ICJpZCIpKSAlPiUNCiAgZmlsdGVyKGF2ZXJhZ2VfZGVwdGggPiAzLjEpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3BpeGVsX3ByZTINCg0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9waXhlbCA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9waXhlbF9wcmUyICU+JQ0KICBuYW1lZF9ncm91cF9zcGxpdChwYTIpDQpgYGANCg0KIyMjIyBSdW4gTW9kZWwNCg0KYGBge3IsIHdhcm5pbmcgPSBmYWxzZX0NCmZpdHNfbm9uX3NlYXJjaF9waXhlbCA8LSBtb2RlbF93cmFwcGVyX2Z1bmN0aW9uKGRmID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfcGl4ZWwscG9seV9pbmQgPSAwKQ0KDQpiZXN0X2luZF9ub25fc2VhcmNoX3BpeGVsIDwtIA0KICBsYXBwbHkoMTpsZW5ndGgoRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfcGl4ZWwpLCBiZXN0X2luZF9mdW5jdGlvbixkZiA9IGZpdHNfbm9uX3NlYXJjaF9waXhlbCwNCiAgICAgICAgIGRmMiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3BpeGVsKSANCg0KY29lZl9ub25fc2VhcmNoX3BpeGVsIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfcGl4ZWwgJT4lIGJpbmRfcm93cyAjbWFrZSBhIG1hdHJpeCBvZiBhbGwgY29lZnMNCg0KYmVzdF9maXRfbm9uX3NlYXJjaF9waXhlbCA8LSBiZXN0X2luZF9ub25fc2VhcmNoX3BpeGVsICU+JQ0KICBzZXRfbmFtZXMobmFtZXNfZnVuY3Rpb24oRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfcGl4ZWxfcHJlLCBwYTIpKSAgDQpgYGANCg0KIyMjIyBDcmVhdGUgR3JhcGggT2JqZWN0DQoNCmBgYHtyfQ0KZ3JhcGhfbGlzdF9waXhlbCA8LSBsYXBwbHkoMTpsZW5ndGgoYmVzdF9maXRfbm9uX3NlYXJjaF9waXhlbCksIGdyYXBoaW5nX2Z1bmN0aW9uNCwgZGYxID0gYmVzdF9maXRfbm9uX3NlYXJjaF9waXhlbCwgZGYyID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfcGl4ZWwpDQpgYGANCg0KDQpgYGB7cn0NCmVuZF90aW1lIDwtIFN5cy50aW1lKCkNCg0KdGltZV9waXhlbCA9IGVuZF90aW1lIC0gc3RhcnRfdGltZQ0KYGBgDQoNCg0KIyMjIEZpDQoNCiMjIyMgRGF0YSBSZWFkaW4NCg0KYGBge3J9DQoNCnN0YXJ0X3RpbWUgPC0gU3lzLnRpbWUoKQ0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9maV9wcmUgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGUgJT4lDQogIGZpbHRlcihwYSA9PSAiR29vZ2xlIEZpIikgJT4lDQogIG11dGF0ZSgNCiAgICBwYTIgPSAiRmkgLSBBbGwgQ2hhbm5lbCINCiAgKSAlPiUNCiAgIyAgIGZpbHRlcihhYnNvbHV0ZV9saWZ0IDwgMTAwMCkgICU+JQ0KICAjIGZpbHRlcihzdHVkeV9pZCAhPSAnNjI5NzQyMCcpICMlPiUNCiAgIyAgZmlsdGVyKHN0dWR5X2lkICE9JzE0OTE2MTcxMScpICU+JQ0KICAjICBmaWx0ZXIoc3R1ZHlfaWQgIT0gJzE0ODYxMzAwMicpICU+JQ0KICAjIGZpbHRlcihzdHVkeV9pZCAhPSczMjg0NjI1JykgJT4lDQogICMgIGZpbHRlcihzdHVkeV9pZCAhPSczMzI5MTMxJykNCiAgbXV0YXRlKA0KICAgIGlkMiA9IHJvd19udW1iZXIoKQ0KICApDQoNCmRmX3Rlc3QgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZmlfcHJlICU+JQ0KICAjIHNlbGVjdCgtc3R1ZHlfaWQsIC1pZDIsIC1yZWdpb24sIC1zY2FsaW5nX2ZhY3RvciwgLXF1YXJ0ZXIsIC1wYSwgLXN0dWR5X25hbWUpDQogIHNlbGVjdCgNCiAgICByZWdpb25fdjIsIGNvdW50cnksIGNoYW5uZWwsIHRhY3RpYywNCiAgICAjIHRyZWF0bWVudF91c2VyX2NvdW50OmNvbnRyb2wsDQogICAgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwOmFic29sdXRlX2xpZnQNCiAgKQ0KDQppc29fZmkgPC0gaXNvbGF0aW9uRm9yZXN0JG5ldyhzYW1wbGVfc2l6ZSA9IG5yb3coZGZfdGVzdCksIG51bV90cmVlcyA9IDEwMDAwLCBzZWVkID0gMTE1MikNCg0KaXNvX2ZpJGZpdChkZl90ZXN0KQ0KDQpzY29yZXNfdHJhaW4gPC0gZGZfdGVzdCAlPiUNCiAgaXNvX2ZpJHByZWRpY3QoKSAlPiUNCiAgYXJyYW5nZShkZXNjKGFub21hbHlfc2NvcmUpKQ0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9maV9wcmUyIDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2ZpX3ByZSAlPiUNCiAgbGVmdF9qb2luKHNjb3Jlc190cmFpbiwgYnkgPSBjKCJpZDIiID0gImlkIikpICU+JQ0KICBmaWx0ZXIoYXZlcmFnZV9kZXB0aCA+IDQuNzUpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2ZpX3ByZTINCg0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9maSA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9maV9wcmUyICU+JQ0KICBuYW1lZF9ncm91cF9zcGxpdChjaGFubmVsKQ0KYGBgDQoNCiMjIyMgUnVuIE1vZGVsDQoNCmBgYHtyLCB3YXJuaW5nID0gZmFsc2V9DQpmaXRzX25vbl9zZWFyY2hfZmkgPC0gbW9kZWxfd3JhcHBlcl9mdW5jdGlvbihkZiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2ZpLHBvbHlfaW5kID0gMCkNCg0KYmVzdF9pbmRfbm9uX3NlYXJjaF9maSA8LSANCiAgbGFwcGx5KDE6bGVuZ3RoKEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2ZpKSwgYmVzdF9pbmRfZnVuY3Rpb24sZGYgPSBmaXRzX25vbl9zZWFyY2hfZmksDQogICAgICAgICBkZjIgPSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9maSkgDQoNCmNvZWZfbm9uX3NlYXJjaF9maSA8LSBiZXN0X2luZF9ub25fc2VhcmNoX2ZpICU+JSBiaW5kX3Jvd3MgI21ha2UgYSBtYXRyaXggb2YgYWxsIGNvZWZzDQoNCmJlc3RfZml0X25vbl9zZWFyY2hfZmkgPC0gYmVzdF9pbmRfbm9uX3NlYXJjaF9maSAlPiUNCiAgc2V0X25hbWVzKG5hbWVzX2Z1bmN0aW9uKEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2ZpX3ByZSwgcGEyKSkgIA0KYGBgDQoNCiMjIyMgQ3JlYXRlIEdyYXBoIE9iamVjdA0KDQpgYGB7cn0NCmdyYXBoX2xpc3RfZmkgPC0gbGFwcGx5KDE6bGVuZ3RoKGJlc3RfZml0X25vbl9zZWFyY2hfZmkpLCBncmFwaGluZ19mdW5jdGlvbjQsIGRmMSA9IGJlc3RfZml0X25vbl9zZWFyY2hfZmksIGRmMiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2ZpKQ0KYGBgDQoNCg0KYGBge3J9DQplbmRfdGltZSA8LSBTeXMudGltZSgpDQoNCnRpbWVfZmkgPSBlbmRfdGltZSAtIHN0YXJ0X3RpbWUNCmBgYA0KDQoNCiMjIyBTTUIgLSBRTGVhZHMNCg0KIyMjIyBEYXRhIFJlYWRpbg0KDQpgYGB7cn0NCg0Kc3RhcnRfdGltZSA8LSBTeXMudGltZSgpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3NtYnFfcHJlIDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlICU+JQ0KICBmaWx0ZXIoZ3JvdXBlZF9jb252ZXJzaW9uID09ICdMZW5hIFEgTGVhZCcpICU+JQ0KICBtdXRhdGUoDQogICAgcGEyID0gIlNNQiAtIFEtTGVhZCINCiAgKSAlPiUNCiAgIyAgIGZpbHRlcihhYnNvbHV0ZV9saWZ0IDwgMTAwMCkgICU+JQ0KICAjIGZpbHRlcihzdHVkeV9pZCAhPSAnNjI5NzQyMCcpICMlPiUNCiAgIyAgZmlsdGVyKHN0dWR5X2lkICE9JzE0OTE2MTcxMScpICU+JQ0KICAjICBmaWx0ZXIoc3R1ZHlfaWQgIT0gJzE0ODYxMzAwMicpICU+JQ0KICAjIGZpbHRlcihzdHVkeV9pZCAhPSczMjg0NjI1JykgJT4lDQogICMgIGZpbHRlcihzdHVkeV9pZCAhPSczMzI5MTMxJykNCiAgbXV0YXRlKA0KICAgIGlkMiA9IHJvd19udW1iZXIoKQ0KICApDQoNCmRmX3Rlc3QgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21icV9wcmUgJT4lDQogICMgc2VsZWN0KC1zdHVkeV9pZCwgLWlkMiwgLXJlZ2lvbiwgLXNjYWxpbmdfZmFjdG9yLCAtcXVhcnRlciwgLXBhLCAtc3R1ZHlfbmFtZSkNCiAgc2VsZWN0KA0KICAgIHJlZ2lvbl92MiwgY291bnRyeSwgY2hhbm5lbCwgdGFjdGljLA0KICAgICMgdHJlYXRtZW50X3VzZXJfY291bnQ6Y29udHJvbCwNCiAgICBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXA6YWJzb2x1dGVfbGlmdA0KICApDQoNCmlzb19zbWJxIDwtIGlzb2xhdGlvbkZvcmVzdCRuZXcoc2FtcGxlX3NpemUgPSBucm93KGRmX3Rlc3QpLCBudW1fdHJlZXMgPSAxMDAwMCwgc2VlZCA9IDExNTIpDQoNCmlzb19zbWJxJGZpdChkZl90ZXN0KQ0KDQpzY29yZXNfdHJhaW4gPC0gZGZfdGVzdCAlPiUNCiAgaXNvX3NtYnEkcHJlZGljdCgpICU+JQ0KICBhcnJhbmdlKGRlc2MoYW5vbWFseV9zY29yZSkpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3NtYnFfcHJlMiA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJxX3ByZSAlPiUNCiAgbGVmdF9qb2luKHNjb3Jlc190cmFpbiwgYnkgPSBjKCJpZDIiID0gImlkIikpICU+JSANCiAgZmlsdGVyKGF2ZXJhZ2VfZGVwdGggPiAxKQ0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJxIDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3NtYnFfcHJlMiAlPiUNCiAgbmFtZWRfZ3JvdXBfc3BsaXQocGEyKQ0KYGBgDQoNCiMjIyMgUnVuIE1vZGVsDQoNCmBgYHtyLCB3YXJuaW5nID0gZmFsc2V9DQpmaXRzX25vbl9zZWFyY2hfc21icSA8LSBtb2RlbF93cmFwcGVyX2Z1bmN0aW9uKGRmID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21icSxwb2x5X2luZCA9IDApDQoNCmJlc3RfaW5kX25vbl9zZWFyY2hfc21icSA8LSANCiAgbGFwcGx5KDE6bGVuZ3RoKEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3NtYnEpLCBiZXN0X2luZF9mdW5jdGlvbixkZiA9IGZpdHNfbm9uX3NlYXJjaF9zbWJxLA0KICAgICAgICAgZGYyID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21icSkgDQoNCmNvZWZfbm9uX3NlYXJjaF9zbWJxIDwtIGJlc3RfaW5kX25vbl9zZWFyY2hfc21icSAlPiUgYmluZF9yb3dzICNtYWtlIGEgbWF0cml4IG9mIGFsbCBjb2Vmcw0KDQpiZXN0X2ZpdF9ub25fc2VhcmNoX3NtYnEgPC0gYmVzdF9pbmRfbm9uX3NlYXJjaF9zbWJxICU+JQ0KICBzZXRfbmFtZXMobmFtZXNfZnVuY3Rpb24oRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21icV9wcmUsIHBhMikpICANCmBgYA0KDQojIyMjIENyZWF0ZSBHcmFwaCBPYmplY3QNCg0KYGBge3J9DQpncmFwaF9saXN0X3NtYnEgPC0gbGFwcGx5KDE6bGVuZ3RoKGJlc3RfZml0X25vbl9zZWFyY2hfc21icSksIGdyYXBoaW5nX2Z1bmN0aW9uNCwgZGYxID0gYmVzdF9maXRfbm9uX3NlYXJjaF9zbWJxLCBkZjIgPSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJxKQ0KYGBgDQoNCg0KYGBge3J9DQplbmRfdGltZSA8LSBTeXMudGltZSgpDQoNCnRpbWVfc21icSA9IGVuZF90aW1lIC0gc3RhcnRfdGltZQ0KYGBgDQoNCg0KIyMjIFNNQiAtIEJMZWFkcw0KDQojIyMjIERhdGEgUmVhZGluDQoNCmBgYHtyfQ0KDQpzdGFydF90aW1lIDwtIFN5cy50aW1lKCkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21iYl9wcmUgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGUgJT4lDQogIGZpbHRlcihwYSA9PSAiU01CIiAmIGdyb3VwZWRfY29udmVyc2lvbiA9PSAnTGVuYSBCIExlYWQnKSAlPiUNCiAgbXV0YXRlKA0KICAgIHBhMiA9ICJTTUIgLSBCLUxlYWQiDQogICkgJT4lDQogICMgICBmaWx0ZXIoYWJzb2x1dGVfbGlmdCA8IDEwMDApICAlPiUNCiAgIyBmaWx0ZXIoc3R1ZHlfaWQgIT0gJzYyOTc0MjAnKSAjJT4lDQogICMgIGZpbHRlcihzdHVkeV9pZCAhPScxNDkxNjE3MTEnKSAlPiUNCiAgIyAgZmlsdGVyKHN0dWR5X2lkICE9ICcxNDg2MTMwMDInKSAlPiUNCiAgIyBmaWx0ZXIoc3R1ZHlfaWQgIT0nMzI4NDYyNScpICU+JQ0KICAjICBmaWx0ZXIoc3R1ZHlfaWQgIT0nMzMyOTEzMScpDQogIG11dGF0ZSgNCiAgICBpZDIgPSByb3dfbnVtYmVyKCkNCiAgKQ0KDQpkZl90ZXN0IDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX3NtYmJfcHJlICU+JQ0KICAjIHNlbGVjdCgtc3R1ZHlfaWQsIC1pZDIsIC1yZWdpb24sIC1zY2FsaW5nX2ZhY3RvciwgLXF1YXJ0ZXIsIC1wYSwgLXN0dWR5X25hbWUpDQogIHNlbGVjdCgNCiAgICByZWdpb25fdjIsIGNvdW50cnksIGNoYW5uZWwsIHRhY3RpYywNCiAgICAjIHRyZWF0bWVudF91c2VyX2NvdW50OmNvbnRyb2wsDQogICAgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwOmFic29sdXRlX2xpZnQNCiAgKQ0KDQppc29fc21iYiA8LSBpc29sYXRpb25Gb3Jlc3QkbmV3KHNhbXBsZV9zaXplID0gbnJvdyhkZl90ZXN0KSwgbnVtX3RyZWVzID0gMTAwMDAsIHNlZWQgPSAxMTUyKQ0KDQppc29fc21iYiRmaXQoZGZfdGVzdCkNCg0Kc2NvcmVzX3RyYWluIDwtIGRmX3Rlc3QgJT4lDQogIGlzb19zbWJiJHByZWRpY3QoKSAlPiUNCiAgYXJyYW5nZShkZXNjKGFub21hbHlfc2NvcmUpKQ0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJiX3ByZTIgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21iYl9wcmUgJT4lDQogIGxlZnRfam9pbihzY29yZXNfdHJhaW4sIGJ5ID0gYygiaWQyIiA9ICJpZCIpKSAlPiUgDQogIGZpbHRlcihhdmVyYWdlX2RlcHRoID4gNCkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21iYiA8LQ0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJiX3ByZTIgJT4lDQogIG5hbWVkX2dyb3VwX3NwbGl0KGNoYW5uZWwpDQpgYGANCg0KIyMjIyBSdW4gTW9kZWwNCg0KYGBge3IsIHdhcm5pbmcgPSBmYWxzZX0NCmZpdHNfbm9uX3NlYXJjaF9zbWJiIDwtIG1vZGVsX3dyYXBwZXJfZnVuY3Rpb24oZGYgPSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJiLHBvbHlfaW5kID0gMCkNCg0KYmVzdF9pbmRfbm9uX3NlYXJjaF9zbWJiIDwtIA0KICBsYXBwbHkoMTpsZW5ndGgoRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfc21iYiksIGJlc3RfaW5kX2Z1bmN0aW9uLGRmID0gZml0c19ub25fc2VhcmNoX3NtYmIsDQogICAgICAgICBkZjIgPSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJiKSANCg0KY29lZl9ub25fc2VhcmNoX3NtYmIgPC0gYmVzdF9pbmRfbm9uX3NlYXJjaF9zbWJiICU+JSBiaW5kX3Jvd3MgI21ha2UgYSBtYXRyaXggb2YgYWxsIGNvZWZzDQoNCmJlc3RfZml0X25vbl9zZWFyY2hfc21iYiA8LSBiZXN0X2luZF9ub25fc2VhcmNoX3NtYmIgJT4lDQogIHNldF9uYW1lcyhuYW1lc19mdW5jdGlvbihGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJiX3ByZSwgY2hhbm5lbCkpICANCmBgYA0KDQojIyMjIENyZWF0ZSBHcmFwaCBPYmplY3QNCg0KYGBge3J9DQpncmFwaF9saXN0X3NtYmIgPC0gbGFwcGx5KDE6bGVuZ3RoKGJlc3RfZml0X25vbl9zZWFyY2hfc21iYiksIGdyYXBoaW5nX2Z1bmN0aW9uNCwgZGYxID0gYmVzdF9maXRfbm9uX3NlYXJjaF9zbWJiLCBkZjIgPSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9zbWJiKQ0KYGBgDQoNCg0KYGBge3J9DQplbmRfdGltZSA8LSBTeXMudGltZSgpDQoNCnRpbWVfc21iYiA9IGVuZF90aW1lIC0gc3RhcnRfdGltZQ0KYGBgDQoNCiMjIEV4cG9ydCBhbGwgZ3JhcGggbGlzdHMNCg0KYGBge3J9DQpncmFwaF9uYW1lcyA8LSBtZ2V0KGxzKHBhdCA9ICdncmFwaF9saXN0XycpKQ0KICAgDQpkZl9uYW1lcyA8LSBtZ2V0KHNldGRpZmYobHMocGF0dGVybiA9ICdGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV8nKSwgbHMocGF0dGVybiA9ICJwcmUiKSkpDQoNCiNybShGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9DaHJvbWUsRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfQ2xvdWQsRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfWW91VHViZSkNCg0KI2xhcHBseSgxOmxlbmd0aChncmFwaF9uYW1lcyksDQojICAgICAgZnVuY3Rpb24oaikgew0KI2xhcHBseSgxOmxlbmd0aChkZl9uYW1lc1tbal1dKSxleHBvcnRfcnBsb3RzX2Z1bmN0aW9uMixzdGFydGluZ19uYW1lID0gIk5vbl9TZWFyY2hfIixmb2xkZXJfbmFtZSA9IGZvbGRlcl9uYW1lLGRmX2xpc3QgPSAjZGZfbmFtZXNbW2pdXSxncmFwaGluZ19saXN0ID0gZ3JhcGhfbmFtZXNbal1bWzFdXSkNCiMgICAgICB9DQojICAgICAgICkNCmBgYA0KDQojIyBHcmlkIG9mIGFsbCBSZXNwb25zZSBDdXJ2ZXMNCg0KWypTdWIgUGxvdCBEb2N1bWVudGF0aW9uKl0oaHR0cHM6Ly9wbG90bHkuY29tL3Ivc3VicGxvdHMvKQ0KDQpgYGB7ciwgZmlnLmhlaWdodD0gMTUsIGVjaG89RkFMU0UsbWVzc2FnZT1GQUxTRSwgd2FybmluZyA9IEZBTFNFfQ0KDQpsYXBwbHkoMTpsZW5ndGgoZ3JhcGhfbmFtZXMpLA0KZnVuY3Rpb24oaSl7DQogIHN1YnBsb3QoZ3JhcGhfbmFtZXNbaV1bWzFdXSwgbnJvd3MgPSBsZW5ndGgoZ3JhcGhfbmFtZXNbaV1bWzFdXSkpDQp9DQopDQoNCg0KbGFwcGx5KDE6bGVuZ3RoKGdyYXBoX25hbWVzKSwNCmZ1bmN0aW9uKGkpew0KI3AxID0gZ3JhcGhfbmFtZXNbaV1bWzFdXQ0KZG8uY2FsbChncmlkLmFycmFuZ2UsZ3JhcGhfbmFtZXNbaV1bWzFdXSkNCiNyZXR1cm4oZ3JpZC5hcnJhbmdlKGdyb2JzID0gcDEpKQ0KfQ0KKQ0KDQpgYGANCg0KIyMgQ29lZiBNYXRyaXgNCg0KYGBge3J9DQpjb2VmLjJfbWF0cml4IDwtIG1nZXQoKGxzKHBhdCA9ICdjb2VmXycpKSkgJT4lICBiaW5kX3Jvd3MoKQ0KDQpjb2VmLjJfbWF0cml4DQoNCg0KYGBgDQoNCiMjIEdyYXBocyB3aXRoIEFub21hbHkgU2NvcmVzDQoNCmBgYHtyfQ0KZ3JhcGhfbGlzdC5maSA8LSBsYXBwbHkoMTpsZW5ndGgoYmVzdF9maXRfbm9uX3NlYXJjaF9maSksIGdyYXBoaW5nX2Z1bmN0aW9uNF93X2Fub20sIGRmMSA9IGJlc3RfZml0X25vbl9zZWFyY2hfZmksIGRmMiA9IEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2ZpKQ0KDQojIyMgQWRkIEdHIFRleHQgUmVwZWwNCmdncGxvdGx5KGdyYXBoX2xpc3QuZmlbWzNdXSkNCg0KYGBgDQoNCiMgQ3JlYXRlIGFsbCBSZXNwb25zZSBDdXJ2ZXMgLSBSaWRnZS9MYXNzbw0KDQpgYGB7ciwgd2FybmluZyA9IEZBTFNFfQ0KDQpzdGFydF90aW1lIDwtIFN5cy50aW1lKCkNCg0KZml0cy5ub24uc2VhcmNoLlJJREdFX0xBU1NPIDwtIGxhcHBseSgNCiAgMTpsZW5ndGgoZGZfbmFtZXMpLA0KICBmdW5jdGlvbihpKSB7DQogICAgbW9kZWxfd3JhcHBlcl9mdW5jdGlvbihkZiA9IGRmX25hbWVzW2ldW1sxXV0scG9seV9pbmQgPSAwKQ0KICB9DQopDQoNCmVuZF90aW1lIDwtIFN5cy50aW1lKCkNCg0KY29tYmluZWRfcmlkZ2VfdGltZSA8LSBzdGFydF90aW1lIC0gZW5kX3RpbWUNCg0KYmVzdC5pbmQubm9uLnNlYXJjaC5SSURHRV9MQVNTTyA8LSBsYXBwbHkoDQogIDE6bGVuZ3RoKGRmX25hbWVzKSwNCiAgZnVuY3Rpb24oaSkgeyAgIA0KICBsYXBwbHkoMTpsZW5ndGgoZGZfbmFtZXNbaV1bWzFdXSksIGJlc3RfaW5kX2Z1bmN0aW9uLGRmID0gZml0cy5ub24uc2VhcmNoLlJJREdFX0xBU1NPW2ldW1sxXV0sDQogICAgICAgICBkZjIgPSBkZl9uYW1lc1tpXVtbMV1dKQ0KICB9DQopDQoNCmNvZWYubm9uLnNlYXJjaC5SSURHRV9MQVNTTyA8LSBsYXBwbHkoDQogIDE6bGVuZ3RoKGRmX25hbWVzKSwNCiAgZnVuY3Rpb24gKGkpew0KICBiZXN0LmluZC5ub24uc2VhcmNoLlJJREdFX0xBU1NPW2ldW1sxXV0gJT4lIGJpbmRfcm93cw0KICB9DQopICU+JQ0KICBiaW5kX3Jvd3MoKSAlPiUgDQogIGFzLmRhdGEuZnJhbWUoKSAlPiUgDQojICBtdXRhdGUoDQojICAgIGNvc3RfcDIgPSAwLA0KIyAgICBsYW1iZGEgPSAwLA0KIyAgICBhbHBoYSA9IDAsDQojICAgIHBvd2VyczIgPSAwDQojICApICU+JSANCiAgc2VsZWN0KG9uZV9vZihjb2xuYW1lcyhjb2VmLjJfbWF0cml4KSkpDQoNCg0KYmVzdC5maXQubm9uLnNlYXJjaC5SSURHRV9MQVNTTyA8LSBsYXBwbHkoMTpsZW5ndGgoZGZfbmFtZXMpLA0KICAgICAgZnVuY3Rpb24oaikgew0KbGFwcGx5KDE6bGVuZ3RoKGJlc3QuaW5kLm5vbi5zZWFyY2guUklER0VfTEFTU09bW2pdXSksDQogICAgICBmdW5jdGlvbihpKXsNCiAgICAgICAgYmVzdC5pbmQubm9uLnNlYXJjaC5SSURHRV9MQVNTT1tqXVtbMV1dW2ldICU+JSANCiAgICAgICAgc2V0X25hbWVzKG5tID0gYmVzdC5pbmQubm9uLnNlYXJjaC5SSURHRV9MQVNTT1tqXVtbMV1dW1tpXV1bIm1vZGVsIl0pDQogICAgICB9IA0KKQ0KICAgICAgfQ0KICAgICAgICkNCiAgDQoNCg0KYGBgDQoNCg0KIyBDcmVhdGUgYWxsIFJlc3BvbnNlIEN1cnZlcyAtIFJMTQ0KDQpgYGB7ciwgd2FybmluZyA9IEZBTFNFfQ0KDQoNCnN0YXJ0X3RpbWUgPC0gU3lzLnRpbWUoKQ0KDQpmaXRzLm5vbi5zZWFyY2guUkxNIDwtIGxhcHBseSgNCiAgMTpsZW5ndGgoZGZfbmFtZXMpLA0KICBmdW5jdGlvbihpKSB7DQogICAgbW9kZWxfd3JhcHBlcl9mdW5jdGlvbjIoZGYgPSBkZl9uYW1lc1tpXVtbMV1dKQ0KICB9DQopDQoNCmVuZF90aW1lIDwtIFN5cy50aW1lKCkNCg0KY29tYmluZWRfcmxtX3RpbWUgPC0gc3RhcnRfdGltZSAtIGVuZF90aW1lDQoNCmJlc3QuaW5kLm5vbi5zZWFyY2guUkxNIDwtIGxhcHBseSgNCiAgMTpsZW5ndGgoZGZfbmFtZXMpLA0KICBmdW5jdGlvbihpKSB7ICAgDQogIGxhcHBseSgxOmxlbmd0aChkZl9uYW1lc1tpXVtbMV1dKSwgYmVzdF9pbmRfZnVuY3Rpb24sZGYgPSBmaXRzLm5vbi5zZWFyY2guUkxNW2ldW1sxXV0sDQogICAgICAgICBkZjIgPSBkZl9uYW1lc1tpXVtbMV1dKQ0KICB9DQopDQoNCmNvZWYubm9uLnNlYXJjaC5STE0gPC0gbGFwcGx5KA0KICAxOmxlbmd0aChkZl9uYW1lcyksDQogIGZ1bmN0aW9uIChpKXsNCiAgYmVzdC5pbmQubm9uLnNlYXJjaC5STE1baV1bWzFdXSAlPiUgYmluZF9yb3dzDQogIH0NCikgJT4lDQogIGJpbmRfcm93cygpICU+JSANCiAgYXMuZGF0YS5mcmFtZSgpICU+JSANCiAgbXV0YXRlKA0KICAgIGNvc3RfcDIgPSAwLA0KICAgIGxhbWJkYSA9IDAsDQogICAgYWxwaGEgPSAwLA0KICAgIHBvd2VyczIgPSAwDQogICkgJT4lIA0KICBzZWxlY3Qob25lX29mKGNvbG5hbWVzKGNvZWYuMl9tYXRyaXgpKSkNCg0KDQpiZXN0LmZpdC5ub24uc2VhcmNoLlJMTSA8LSBsYXBwbHkoMTpsZW5ndGgoZGZfbmFtZXMpLA0KICAgICAgZnVuY3Rpb24oaikgew0KbGFwcGx5KDE6bGVuZ3RoKGJlc3QuaW5kLm5vbi5zZWFyY2guUkxNW1tqXV0pLA0KICAgICAgZnVuY3Rpb24oaSl7DQogICAgICAgIGJlc3QuaW5kLm5vbi5zZWFyY2guUkxNW2pdW1sxXV1baV0gJT4lIA0KICAgICAgICBzZXRfbmFtZXMobm0gPSBiZXN0LmluZC5ub24uc2VhcmNoLlJMTVtqXVtbMV1dW1tpXV1bIm1vZGVsIl0pDQogICAgICB9IA0KKQ0KICAgICAgfQ0KICAgICAgICkNCg0KDQogIA0KYGBgDQoNCg0KYGBge3J9DQoNCi1jb21iaW5lZF9yaWRnZV90aW1lK2NvbWJpbmVkX3JsbV90aW1lDQoNCmBgYA0KDQoNCmBgYHtyLCB3YXJuaW5nID0gRkFMU0V9DQoNCmdyYXBoLmxpc3QucmxtIDwtIGxhcHBseSgxOmxlbmd0aChkZl9uYW1lcyksDQogICAgICBmdW5jdGlvbihpKXsNCiAgICAgIGxhcHBseSgxOmxlbmd0aChiZXN0LmZpdC5ub24uc2VhcmNoLlJMTVtpXVtbMV1dKSwgZ3JhcGhpbmdfZnVuY3Rpb25fcmxtLCBkZjE9IGJlc3QuZml0Lm5vbi5zZWFyY2guUkxNW2ldLGRmMiA9IGRmX25hbWVzW2ldKQ0KICAgICAgfSANCikNCg0KDQpncmFwaC5saXN0LlJJREdFX0xBU1NPIDwtIGxhcHBseSgxOmxlbmd0aChkZl9uYW1lcyksDQogICAgICBmdW5jdGlvbihpKXsNCiAgICAgIGxhcHBseSgxOmxlbmd0aChiZXN0LmZpdC5ub24uc2VhcmNoLlJJREdFX0xBU1NPW2ldW1sxXV0pLCBncmFwaGluZ19mdW5jdGlvbl9lbGFzdGljbmV0LCBkZjE9IGJlc3QuZml0Lm5vbi5zZWFyY2guUklER0VfTEFTU09baV0sZGYyID0gZGZfbmFtZXNbaV0pDQogICAgICB9IA0KKQ0KDQoNCg0KDQpgYGANCg0KDQojIyMgRXhwb3J0IGFsbCBQbG90cw0KDQpgYGB7cn0NCg0KZm9sZGVyX25hbWUxIDwtIHBhc3RlMCgiT3V0cHV0LyIsICJvdXRwdXRmaWxlc18iLCBTeXMuRGF0ZSgpLCAiXyIsICJSTE0iLCAiLyIpDQpkaXIuY3JlYXRlKGZvbGRlcl9uYW1lMSkgIyBpdCB3aWxsIHRocm93IGEgd2FybmluZyBpZiBmb2xkZXIgZXhpc3RzDQoNCmxhcHBseSgxOmxlbmd0aChkZl9uYW1lcyksDQogICAgICBmdW5jdGlvbihqKSB7DQpsYXBwbHkoMTpsZW5ndGgoZGZfbmFtZXNbW2pdXSksZXhwb3J0X3JwbG90c19mdW5jdGlvbjIsc3RhcnRpbmdfbmFtZSA9ICJOb25fU2VhcmNoXyIsZm9sZGVyX25hbWUgPSBmb2xkZXJfbmFtZTEsZGZfbGlzdCA9IGRmX25hbWVzW1tqXV0sZ3JhcGhpbmdfbGlzdCA9IGdyYXBoLmxpc3QucmxtW2pdW1sxXV0pDQogICAgICB9DQogICAgICAgKQ0KDQpmb2xkZXJfbmFtZTIgPC0gcGFzdGUwKCJPdXRwdXQvIiwgIm91dHB1dGZpbGVzXyIsIFN5cy5EYXRlKCksICJfIiwgIkVsYXN0aWNOZXQiLCAiLyIpDQpkaXIuY3JlYXRlKGZvbGRlcl9uYW1lMikgIyBpdCB3aWxsIHRocm93IGEgd2FybmluZyBpZiBmb2xkZXIgZXhpc3RzDQoNCg0KbGFwcGx5KDE6bGVuZ3RoKGRmX25hbWVzKSwNCiAgICAgIGZ1bmN0aW9uKGopIHsNCmxhcHBseSgxOmxlbmd0aChkZl9uYW1lc1tbal1dKSxleHBvcnRfcnBsb3RzX2Z1bmN0aW9uMixzdGFydGluZ19uYW1lID0gIk5vbl9TZWFyY2hfIixmb2xkZXJfbmFtZSA9IGZvbGRlcl9uYW1lMixkZl9saXN0ID0gZGZfbmFtZXNbW2pdXSxncmFwaGluZ19saXN0ID0gZ3JhcGgubGlzdC5SSURHRV9MQVNTT1tqXVtbMV1dKQ0KICAgICAgfQ0KICAgICAgICkNCg0KYGBgDQoNCiMjIyBTaG93IEdyYXBocw0KDQpgYGB7cn0NCmxhcHBseSgxOmxlbmd0aChkZl9uYW1lcyksDQpmdW5jdGlvbihqKXsNCiAgc3VicGxvdChncmFwaC5saXN0LnJsbVtqXVtbMV1dLCBucm93cyA9IGxlbmd0aChncmFwaC5saXN0LnJsbVtqXVtbMV1dKSkNCn0NCikNCg0KDQpsYXBwbHkoMTpsZW5ndGgoZGZfbmFtZXMpLA0KZnVuY3Rpb24oaSl7DQojcDEgPSBncmFwaF9uYW1lc1tpXVtbMV1dDQpkby5jYWxsKGdyaWQuYXJyYW5nZSxncmFwaC5saXN0LnJsbVtpXVtbMV1dKQ0KI3JldHVybihncmlkLmFycmFuZ2UoZ3JvYnMgPSBwMSkpDQp9DQopDQoNCg0KDQpgYGANCg0KYGBge3J9DQpsYXBwbHkoMTpsZW5ndGgoZGZfbmFtZXMpLA0KZnVuY3Rpb24oail7DQogIHN1YnBsb3QoZ3JhcGgubGlzdC5SSURHRV9MQVNTT1tqXVtbMV1dLCBucm93cyA9IGxlbmd0aChncmFwaC5saXN0LlJJREdFX0xBU1NPW2pdW1sxXV0pKQ0KfQ0KKQ0KDQoNCmxhcHBseSgxOmxlbmd0aChkZl9uYW1lcyksDQpmdW5jdGlvbihpKXsNCiNwMSA9IGdyYXBoX25hbWVzW2ldW1sxXV0NCmRvLmNhbGwoZ3JpZC5hcnJhbmdlLGdyYXBoLmxpc3QuUklER0VfTEFTU09baV1bWzFdXSkNCiNyZXR1cm4oZ3JpZC5hcnJhbmdlKGdyb2JzID0gcDEpKQ0KfQ0KKQ0KDQpgYGANCg0KDQojIFRlc3RpbmcgTWV0YWZvciBQYWNrYWdlDQoNCmBgYHtyfQ0KDQpwX2xvYWQobG1lNCkNCnBfbG9hZChtZXRhZm9yZXN0KQ0KDQpgYGANCg0KIyMgVGVzdGluZyBvbiBEU00gRGF0YQ0KDQojIyMgTG9hZCBpbiBEYXRhDQpgYGB7cn0NCg0KDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9kc21fcHJlIDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlICU+JQ0KICBmaWx0ZXIocGEgPT0gIkRTTSIpICU+JQ0KICBmaWx0ZXIocmVnaW9uX3YyICE9ICJBUEFDIikgJT4lDQogICMgZmlsdGVyKGFic29sdXRlX2xpZnQgPCAxMDAwKSAjICU+JQ0KICAjIGZpbHRlcihzdHVkeV9pZCAhPSAnNjI5NzQyMCcpICMlPiUNCiAgIyAgZmlsdGVyKHN0dWR5X2lkICE9JzE0OTE2MTcxMScpICU+JQ0KICAjICBmaWx0ZXIoc3R1ZHlfaWQgIT0gJzE0ODYxMzAwMicpICU+JQ0KICAjIGZpbHRlcihzdHVkeV9pZCAhPSczMjg0NjI1JykgJT4lDQogICMgIGZpbHRlcihzdHVkeV9pZCAhPSczMzI5MTMxJykNCiAgbXV0YXRlKA0KICAgIGlkMiA9IHJvd19udW1iZXIoKQ0KICApDQoNCmRmX3Rlc3QgPC0NCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtX3ByZSAlPiUNCiAgIyBzZWxlY3QoLXN0dWR5X2lkLCAtaWQyLCAtcmVnaW9uLCAtc2NhbGluZ19mYWN0b3IsIC1xdWFydGVyLCAtcGEsIC1zdHVkeV9uYW1lKQ0KICBzZWxlY3QoDQogICAgcmVnaW9uX3YyLCBjb3VudHJ5LCBjaGFubmVsLCB0YWN0aWMsDQogICAgIyB0cmVhdG1lbnRfdXNlcl9jb3VudDpjb250cm9sLA0KICAgIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cDphYnNvbHV0ZV9saWZ0DQogICkNCg0KaXNvX2RzbSA8LSBpc29sYXRpb25Gb3Jlc3QkbmV3KHNhbXBsZV9zaXplID0gbnJvdyhkZl90ZXN0KSwgbnVtX3RyZWVzID0gMTAwMDAsIHNlZWQgPSAxMTUyKQ0KDQppc29fZHNtJGZpdChkZl90ZXN0KQ0KDQpzY29yZXNfdHJhaW4gPC0gZGZfdGVzdCAlPiUNCiAgaXNvX2RzbSRwcmVkaWN0KCkgJT4lDQogIGFycmFuZ2UoZGVzYyhhbm9tYWx5X3Njb3JlKSkNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtX01ldGE0X1YxIDwtDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2RzbV9wcmUgJT4lDQogIGxlZnRfam9pbihzY29yZXNfdHJhaW4sIGJ5ID0gYygiaWQyIiA9ICJpZCIpKSAlPiUNCiBmaWx0ZXIoYXZlcmFnZV9kZXB0aCA+IDUuMjA5KQ0KDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2RzbV9NZXRhNCA8LQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtX01ldGE0X1YxICU+JSANCiAgbXV0YXRlKA0KICAgIHAxID0gZXhwb3NlZC90cmVhdG1lbnRfdXNlcl9jb3VudCwNCiAgICBxMSA9IDEgLSBwMSwNCiAgICBuMSA9IHRyZWF0bWVudF91c2VyX2NvdW50LA0KICAgIHNkMSA9IHNxcnQocDEqcTEqbjEpLA0KICAgIHAyID0gc2NhbGVkX2NvbnRyb2wvdHJlYXRtZW50X3VzZXJfY291bnQsDQogICAgcTIgPSAxLXAyLA0KICAgIG4yID0gdHJlYXRtZW50X3VzZXJfY291bnQsIA0KICAgIHNkMiA9IHNxcnQocDIqcTIqbjIpLA0KICAgIGNvc3RfcCA9IGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCBeIDAuNA0KICApICU+JSANCiAgc2VsZWN0KC1wMSwtcTEsLW4xLC1wMiwtcTIsLW4yKSAlPiUgDQogIG5hbWVkX2dyb3VwX3NwbGl0KHJlZ2lvbl92MikNCg0KYGBgDQoNCiMjIyBTdGFuZGFyZGl6ZSBEYXRhIGFuZCBhZnRlciBjYWxjdWxhdGluZyBzdGFuZGFyZCBkZXZpYXRpb24NCg0KYGBge3J9DQoNCmRmX1NNRCA8LSBsaXN0KCkNCmZvciAoaSBpbiAxOmxlbmd0aChGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X05vbl9TZWFyY2hfbW9kZWxfZmlsZV9kc21fTWV0YTQpKXsNCiAgZGZfU01EW1tpXV0gPC0NCiAgZXNjYWxjKA0KICAgIG1lYXN1cmUgPSAiU01EIiwNCiAgICAgICAgICAgICAgICAgbTFpID0gZXhwb3NlZCwNCiAgICAgICAgICAgICAgICAgbTJpID0gc2NhbGVkX2NvbnRyb2wsDQogICAgICAgICAgICAgICAgIHNkMWkgPSBzZDEsDQogICAgICAgICAgICAgICAgIHNkMmkgPSBzZDIsDQogICAgICAgICAgICAgICAgIG4xaSA9IHRyZWF0bWVudF91c2VyX2NvdW50LA0KICAgICAgICAgICAgICAgICBuMmkgPSB0cmVhdG1lbnRfdXNlcl9jb3VudCwNCiAgICBkYXRhID0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtX01ldGE0W1tpXV0NCiAgKQ0KICBuYW1lcyhkZl9TTUQpW2ldIDwtIG5hbWVzKEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfTm9uX1NlYXJjaF9tb2RlbF9maWxlX2RzbV9NZXRhNFtpXSkNCn0NCmBgYA0KDQoNCg0KIyMjIFJ1biBhIEZpeGVkLUVmZmVjdHMgTW9kZWwNCltEb2N1bWVudGF0aW9uXShodHRwczovL2NqdmFubGlzc2EuZ2l0aHViLmlvL0RvaW5nLU1ldGEtQW5hbHlzaXMtaW4tUi9maXhlZGVmLmh0bWwpDQpgYGB7ciwgZmlnLmhlaWdodD0gMTV9DQoNCg0KDQppID0gMQ0KDQp5aV9EU00gPSBkZl9TTURbW2ldXVsnYWJzb2x1dGVfbGlmdCddICU+JSB1bmxpc3QoKQ0KdmlfRFNNID0gZGZfU01EW1tpXV1bJ3NkMSddICU+JSB1bmxpc3QoKQ0Kc3BsaXQyID0gZmFjdG9yKGRmX1NNRFtbaV1dWydjaGFubmVsJ10gJT4lIHVubGlzdCgpLGxhYmVscyA9IHVuaXF1ZShkZl9TTURbW2ldXVsnY2hhbm5lbCddKSAlPiUgdW5saXN0KCkpDQoNCg0KbV9yZWcgPC0gcm1hKHlpID0geWksICAgICAjIFRoZSBkLWNvbHVtbiBvZiB0aGUgZGYsIHdoaWNoIGNvbnRhaW5zIENvaGVuJ3MgZA0KICAgICAgICAgdmkgPSB2aSAgICMgVGhlIHZpLWNvbHVtbiBvZiB0aGUgZGYsIHdoaWNoIGNvbnRhaW5zIHRoZSB2YXJpYW5jZXMNCiAgICAgICAsbW9kcyA9IH5jaGFubmVsOmNvc3RfcC0xICN0byByZW1vdmUgaW50ZXJjZXB0IGJldHdlZW4gc2xvcGVzDQogICAgICAgLGRhdGEgPSBkZl9TTURbW2ldXQ0KICAgICAgICAgKSAgDQogICAgICAgDQptX3JlZw0KDQojcm0oRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9Ob25fU2VhcmNoX21vZGVsX2ZpbGVfZHNtX01ldGE0X1YxKSMsbW9kMV90ZXN0LCBpKQ0KDQpwcmVkaWN0KG1fcmVnKQ0KDQoNCmZvcmVzdChtX3JlZywgc2xhYiA9IGRmX1NNRFtbaV1dWydzdHVkeV9uYW1lJ10gJT4lIHVubGlzdCgpLCBhZGRjcmVkID0gVFJVRSkNCmBgYA0KIyMjIEFkZGl0aW9uYWwgVGVzdA0KDQpgYGB7cn0NCg0KIyBTcGVjaWZ5IGJhc2ljIHBsb3QsIG1hcHBpbmcgc2V4IHRvIHRoZSB4LWF4aXMsIGVmZmVjdCBzaXplICdkJyB0byB0aGUgeS1heGlzLA0KIyBhbmQgJ3dlaWdodHMnIHRvIHRoZSB3ZWlnaHQgcGFyYW1ldGVyLg0KDQpkZl9TTURbW2ldXSAlPiUgDQogIGdncGxvdCgpKw0KICBhZXMoDQogICAgeCA9IGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwNCiAgICB5ID0geWksDQogICAgc2l6ZSA9IDEvc3FydCh2aSkNCiAgKSArDQogIGdlb21fcG9pbnQoc2hhcGUgPSAxKSArICMgQWRkIHNjYXR0ZXINCiAgZ2VvbV9hYmxpbmUoaW50ZXJjZXB0ID0gMCwgc2xvcGUgPSBtX3JlZyRiWzJdKSArICMgQWRkIHJlZ3Jlc3Npb24gbGluZQ0KICMgdGhlbWVfYncoKSArICMgQXBwbHkgYmxhY2sgYW5kIHdoaXRlIHRoZW1lDQogIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIikgIyBSZW1vdmUgbGVnZW5kDQoNCmBgYA0KDQoNCiMjIyBNaXhlZCBFZmZlY3RzIE1vZGVsDQoNCkRvY3VtZW50YXRpb246DQoqIGh0dHBzOi8vcGFnZXMuc3RhdC53aXNjLmVkdS9+YmF0ZXMvVXNlUjIwMDgvV29ya3Nob3BELnBkZg0KDQpgYGB7cn0NCg0KaSA9IDENCg0KbW9kX2xtZTQgPC0gbG1lcihmb3JtdWxhID0gbG9nKGFic29sdXRlX2xpZnQpIH4gMCArIGNvc3RfcA0KICAgICAgICMgICAgICAgICAgKyByZWdpb25fdjINCiAgICAgICAgICAgICAjICAgICArIGNoYW5uZWwgDQogICAgIyAgICAgICAgICAgICAgICsgKDArIHJlZ2lvbl92MnxjaGFubmVsKQ0KICAgICAgICAgICAgICAgICAgICsgKDEgKyBjb3N0X3B8Y2hhbm5lbCkgDQogICAgICMgICAgICAgICAgICAgICsgKDArIDF8Y2hhbm5lbCkNCiAgICAgICAgICMgICAgICAgICAgKyAoMSsgMXxjaGFubmVsOnRhY3RpYykNCiAgICAgICMgICAgICAgICAgICAgKyAoY2hhbm5lbHx0YWN0aWMpDQogICAgIyAgICAgICAgICAgICAgICsoY29zdF9wOmNoYW5uZWwpDQogICAgICAgICAgICAgICAsZGF0YSA9IGRmX1NNRFtbaV1dLCBSRU1MID0gIFRSVUUpICNGYWxzZSBjYWxscyBvbiBNTEUgd2hpY2ggYXJlIGtub3duIHRvIGJlIGJpYXNlZA0KDQoNCnN1bW1hcnkobW9kX2xtZTQpDQoNCnByZWRpY3QobW9kX2xtZTQpDQoNCmRmX1NNRFtbaV1dJHByZWRzIDwtIHByZWRpY3QobW9kX2xtZTQpDQoNCmZpeGVmKG1vZF9sbWU0KQ0KcmFuZWYobW9kX2xtZTQsIGRyb3AgPSBGQUxTRSkNCmBgYA0KDQpgYGB7cn0NCnA8LQ0KZGZfU01EW1tpXV0gJT4lICAgDQogIGdncGxvdChhZXMoeD1jb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHk9cHJlZHMsIGdyb3VwID0gY2hhbm5lbCwgY29sb3VyID0gY2hhbm5lbCkpICsNCiAgZ2VvbV9saW5lKCkgKyANCiAgbGFicyh4PSJTcGVuZCIsIHk9IkFic29sdXRlIExpZnQiKSArDQogIGdndGl0bGUoIk1peGVkIEVmZmVjdHMgTW9kZWwiKSArIA0KIyAgc2NhbGVfY29sb3VyX2Rpc2NyZXRlKCdwYScpKw0KICBnZW9tX2ppdHRlcihhZXMoeD1jb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHkgPSBsb2coYWJzb2x1dGVfbGlmdCksIHNpemUgPSB2aSApKSANCg0KcA0KDQpnZ3Bsb3RseShwKQ0KDQpgYGANCg0KDQoNCiMjIyBVcGRhdGVkIFBsb3R0aW5nIEZ1bmN0aW9uDQpodHRwczovL2xtdWRnZTEzLmdpdGh1Yi5pby9zYW1wbGVfY29kZS9taXhlZF9lZmZlY3RzLmh0bWwNCmBgYHtyfQ0KcF9sb2FkKHNqUGxvdCkgI2ZvciBwbG90dGluZyBsbWVyIGFuZCBnbG1lciBtb2RzDQpwX2xvYWQoc2ptaXNjKSANCnBfbG9hZChlZmZlY3RzKQ0KcF9sb2FkKHNqc3RhdHMpICN1c2UgZm9yIHIyIGZ1bmN0aW9ucw0KDQoNCnNqUGxvdDo6cGxvdF9tb2RlbChtb2RfbG1lNCkNCnNqUGxvdDo6IHRhYl9tb2RlbChtb2RfbG1lNCkNCg0KYGBgDQoNCmBgYHtyfQ0KZWZmZWN0c19jb3N0cCA8LSBlZmZlY3RzOjplZmZlY3QodGVybT0gImNvc3RfcCIsIG1vZD0gbW9kX2xtZTQpICU+JSBhcy5kYXRhLmZyYW1lKCkNCnN1bW1hcnkoZWZmZWN0c19jb3N0cCkgI291dHB1dCBvZiB3aGF0IHRoZSB2YWx1ZXMgYXJlDQoNCmBgYA0KDQpgYGB7cn0NCiAgZ2dwbG90KCkgKyANCiAgIzINCiAgZ2VvbV9wb2ludChkYXRhPWRmX1NNRFtbaV1dLCBhZXMoY29zdF9wLCBsb2coYWJzb2x1dGVfbGlmdCkpKSArIA0KICAjMw0KICBnZW9tX3BvaW50KGRhdGE9ZWZmZWN0c19jb3N0cCwgYWVzKHg9Y29zdF9wLCB5PWZpdCksIGNvbG9yPSJibHVlIikgKw0KICAjNA0KICBnZW9tX2xpbmUoZGF0YT1lZmZlY3RzX2Nvc3RwLCBhZXMoeD1jb3N0X3AsIHk9Zml0KSwgY29sb3I9ImJsdWUiKSArDQogICM1DQogIGdlb21fcmliYm9uKGRhdGE9IGVmZmVjdHNfY29zdHAsIGFlcyh4PWNvc3RfcCwgeW1pbj1sb3dlciwgeW1heD11cHBlciksIGFscGhhPSAwLjMsIGZpbGw9ImJsdWUiKSArDQogICM2DQogIGxhYnMoeD0iY29zdF9wIiwgeT0iTG9nKEFic29sdXRlIExpZnQpIikNCmBgYA0KDQoNCg0K